Notes

RSS feed

Calculating KL divergence in closed-form versus Monte Carlo estimation

Comments

Preamble
In [1]:
%matplotlib notebook
In [2]:
import numpy as np
import pandas as pd

import matplotlib.pyplot as plt
import seaborn as sns

from scipy.stats import norm

from keras import backend as K
from keras.layers import (Input, Activation, Dense, Lambda, Layer, 
                          add, multiply)
from keras.models import Model, Sequential
from keras.callbacks import TerminateOnNaN
from keras.datasets import mnist

from tqdm import tnrange

import tensorflow as tf

Using TensorFlow backend.
Notebook Environment
In [3]:
plt.style.use('seaborn-notebook')
sns.set_context('notebook')
np.set_printoptions(precision=2,
                    edgeitems=3,
                    linewidth=80,
                    suppress=True)
In [4]:
'TensorFlow version: ' + K.tf.__version__
Out[4]:
'TensorFlow version: 1.4.0'
In [5]:
sess = tf.InteractiveSession()

Example: Diagonal Gaussians

In [6]:
D = 2
q_mu = np.float32([ 1., 4.])
p_mu = np.float32([-3., 2.])
q_sigma = np.ones(D).astype('float32')
p_sigma = 2.5*np.ones(D).astype('float32')
In [7]:
q = tf.distributions.Normal(loc=q_mu, scale=q_sigma)
p = tf.distributions.Normal(loc=p_mu, scale=p_sigma)
In [8]:
fig, ax = plt.subplots(figsize=(5, 5))

ax.scatter(*p.sample(sample_shape=(500,)).eval().T, 
           s=8., alpha=.8, label='p samples')

ax.scatter(*q.sample(sample_shape=(500,)).eval().T, 
           s=8.,alpha=.8, label='q samples')

ax.set_xlabel('$x_1$')
ax.set_ylabel('$x_2$')

ax.legend()

plt.show()

Closed-form KL divergence between diagonal Gaussians

In [9]:
def kl_divergence_gaussians(q_mu, q_sigma, p_mu, p_sigma):
    
    r = q_mu - p_mu
   
    return np.sum(np.log(p_sigma) - np.log(q_sigma)
                  - .5 * (1. - (q_sigma**2 + r**2) / p_sigma**2), 
                  axis=-1)
In [10]:
kl_true = kl_divergence_gaussians(q_mu, q_sigma, p_mu, p_sigma)
kl_true
Out[10]:
2.5925815
In [11]:
tf.reduce_sum(tf.distributions.kl_divergence(
    tf.distributions.Normal(loc=q_mu, scale=q_sigma),
    tf.distributions.Normal(loc=p_mu, scale=p_sigma)), axis=-1).eval()
Out[11]:
2.5925813

Monte Carlo estimation

The KL divergence is an expectation of log density ratios over distribution p. We can approximate it with Monte Carlo samples.

In [12]:
mc_samples = 10000
In [13]:
def log_density_ratio_gaussians(z, q_mu, q_sigma, p_mu, p_sigma):
 
    r_p = (z - p_mu) / p_sigma
    r_q = (z - q_mu) / q_sigma
    
    return np.sum(np.log(p_sigma) - np.log(q_sigma) + 
                  .5 * (r_p**2 - r_q**2), axis=-1)
In [14]:
mc_estimates = pd.Series(
    log_density_ratio_gaussians(
        q.sample(sample_shape=(mc_samples,)).eval(), 
        q_mu, q_sigma, 
        p_mu, p_sigma
    )
)
In [15]:
# cumulative mean of the MC estimates
mean = mc_estimates.expanding().mean()
In [16]:
golden_size = lambda width: (width, 2. * width / (1 + np.sqrt(5)))
In [17]:
fig, ax = plt.subplots(figsize=golden_size(6))

mean.plot(ax=ax, label='Estimated')

ax.axhline(y=kl_true, color='r', linewidth=2., label='True')

ax.set_xlabel('Monte Carlo samples')
ax.set_ylabel('KL Divergence')

ax.set_ylim(.95*kl_true, 1.05*kl_true)

ax.legend()

plt.show()
In [18]:
fig, ax = plt.subplots(figsize=golden_size(6))

ax.plot(np.square(mean - kl_true))

ax.set_xlabel('Monte Carlo samples')
ax.set_ylabel('Squared error')

ax.set_ylim(-0.01, 1.)

plt.show()
In [19]:
mean.index.name = 'samples'
mean_df = pd.DataFrame(mean.rename('mc_estimate'))
In [20]:
g = sns.JointGrid(x='samples', y='mc_estimate', 
                  data=mean_df.reset_index())

g = g.plot_joint(plt.plot)
g = g.plot_marginals(sns.kdeplot, shade=True)

g.ax_marg_x.clear()
g.ax_marg_x.set_xticks([])
g.ax_marg_x.set_yticks([])

g.set_axis_labels('Monte Carlo samples', 'KL Divergence')
Out[20]:
<seaborn.axisgrid.JointGrid at 0x7f9a6802be80>

Bayesian Sparse Logistic Regression with Spike-and-Slab Priors (using Edward)

Comments

In [1]:
%matplotlib notebook
In [37]:
import numpy as np
import tensorflow as tf
import edward as ed
import matplotlib.pyplot as plt
import seaborn as sns

from edward.models import (Bernoulli, Laplace, Normal, 
                           Empirical, PointMass)

from sklearn.model_selection import train_test_split
from sklearn.datasets import make_moons
In [3]:
plt.style.use('seaborn-notebook')
sns.set_context('notebook')
np.set_printoptions(precision=2,
                    edgeitems=3,
                    linewidth=80,
                    suppress=True)
In [4]:
'TensorFlow version: ' + tf.__version__
Out[4]:
'TensorFlow version: 1.4.0'
Constants
In [5]:
T = 10000  # HMC nbr. of samples
P = 1000  # Nbr. posterior (predictive) samples

seed = 8888
rng = np.random.RandomState(seed)

Toy Dataset ("Moons")

In [6]:
X_train, X_test, y_train, y_test = train_test_split(
    *make_moons(500, random_state=rng),
    test_size=.2,
    random_state=rng
)
In [7]:
N, D = X_train.shape
In [8]:
fig, ax = plt.subplots(figsize=(7, 5))

cb = ax.scatter(*X_test.T, c=y_test, cmap='coolwarm')
fig.colorbar(cb, ax=ax)

ax.set_xlabel('$x_1$')
ax.set_ylabel('$x_2$')

plt.show()

Gaussian Prior

Model specification

In [9]:
X = tf.placeholder(tf.float32, [N, D])

w = Normal(loc=tf.zeros(D), scale=tf.ones(D))
b = Normal(loc=tf.zeros([]), scale=tf.ones([]))

y = Bernoulli(logits=ed.dot(X, w) + b)

Inference

Maximum a posterior (MAP)
In [10]:
qw = PointMass(params=tf.Variable(tf.random_normal([D])))
qb = PointMass(params=tf.Variable(tf.random_normal([])))
In [11]:
inference = ed.MAP({w: qw, b: qb}, data={X: X_train, y: y_train})
inference.run()

1000/1000 [100%] ██████████████████████████████ Elapsed: 2s | Loss: 113.618
In [12]:
w_map = qw.eval()
w_map
Out[12]:
array([ 1.15, -4.49], dtype=float32)
In [13]:
b_map = qb.eval()
b_map
Out[13]:
0.44861546
Hybrid Monte Carlo
In [14]:
qw = Empirical(params=tf.Variable(tf.random_normal([T, D])))
qb = Empirical(params=tf.Variable(tf.random_normal([T])))
In [15]:
inference = ed.HMC({w: qw, b: qb}, data={X: X_train, y: y_train})
inference.initialize(n_print=10, step_size=0.15)

tf.global_variables_initializer().run()

for i in range(inference.n_iter):

    info_dict = inference.update()
    inference.print_progress(info_dict)
    
inference.finalize()

10000/10000 [100%] ██████████████████████████████ Elapsed: 21s | Acceptance Rate: 0.903
In [16]:
w1_min, w1_max = -.2, 2
w2_min, w2_max = -8, 1.

Evaluation

In [17]:
g = sns.JointGrid(*qw.sample(sample_shape=P).eval().T,
                  xlim=(w1_min, w1_max),
                  ylim=(w2_min, w2_max))
g = g.plot_joint(plt.scatter, s=3.**2, alpha=.4)
g = g.plot_joint(sns.kdeplot, zorder=0, n_levels=6)
g = g.plot_marginals(sns.distplot, kde=True)
g = g.set_axis_labels('$w_1$', '$w_2$')

Laplace Prior

Model specification

In [18]:
X = tf.placeholder(tf.float32, [N, D])

w = Laplace(loc=tf.zeros(D), scale=tf.ones(D))
b = Laplace(loc=tf.zeros([]), scale=tf.ones([]))

y = Bernoulli(logits=ed.dot(X, w) + b)

Inference

Maximum a posterior (MAP)
In [19]:
qw = PointMass(params=tf.Variable(tf.random_normal([D])))
qb = PointMass(params=tf.Variable(tf.random_normal([])))
In [20]:
inference = ed.MAP({w: qw, b: qb}, data={X: X_train, y: y_train})
inference.run()

1000/1000 [100%] ██████████████████████████████ Elapsed: 1s | Loss: 106.517
In [21]:
w_map = qw.eval()
w_map
Out[21]:
array([ 1.17, -5.56], dtype=float32)
In [22]:
b_map = qb.eval()
b_map
Out[22]:
0.68323755
Hybrid Monte Carlo
In [23]:
qw = Empirical(params=tf.Variable(tf.random_normal([T, D])))
qb = Empirical(params=tf.Variable(tf.random_normal([T])))
In [24]:
inference = ed.HMC({w: qw, b: qb}, data={X: X_train, y: y_train})
inference.initialize(n_print=10, step_size=0.15)

tf.global_variables_initializer().run()

for i in range(inference.n_iter):

    info_dict = inference.update()
    inference.print_progress(info_dict)
    
inference.finalize()

10000/10000 [100%] ██████████████████████████████ Elapsed: 23s | Acceptance Rate: 0.910

Evaluation

In [25]:
g = sns.JointGrid(*qw.sample(sample_shape=P).eval().T,
                  xlim=(w1_min, w1_max),
                  ylim=(w2_min, w2_max))
g = g.plot_joint(plt.scatter, s=3.**2, alpha=.4)
g = g.plot_joint(sns.kdeplot, zorder=0, n_levels=6)
g = g.plot_marginals(sns.distplot, kde=True)
g = g.set_axis_labels('$w_1$', '$w_2$')

Spike-and-Slab Prior

Model specification

In [26]:
X = tf.placeholder(tf.float32, [N, D])

w_mask = Bernoulli(logits=tf.zeros(D))
w_raw = Normal(loc=tf.zeros(D), scale=tf.ones(D))
w = tf.to_float(w_mask) * w_raw

b_mask = Bernoulli(logits=tf.zeros([]))
b_raw = Normal(loc=tf.zeros([]), scale=tf.ones([]))
b = tf.to_float(b_mask) * b_raw

y = Bernoulli(logits=ed.dot(X, w) + b)

Inference

Maximum a posterior (MAP)
In [27]:
qw = PointMass(params=tf.Variable(tf.random_normal([D])))
qb = PointMass(params=tf.Variable(tf.random_normal([])))
In [28]:
inference = ed.MAP({w_raw: qw, b_raw: qb}, 
                   data={X: X_train, y: y_train})
inference.run()

1000/1000 [100%] ██████████████████████████████ Elapsed: 2s | Loss: 286.974
In [29]:
w_map = qw.eval()
w_map
Out[29]:
array([ 1.3 , -3.53], dtype=float32)
In [30]:
b_map = qb.eval()
b_map
Out[30]:
-0.046046846
Hybrid Monte Carlo
In [31]:
qw = Empirical(params=tf.Variable(tf.random_normal([T, D])))
qb = Empirical(params=tf.Variable(tf.random_normal([T])))
In [32]:
inference = ed.HMC({w_raw: qw, b_raw: qb}, 
                   data={X: X_train, y: y_train})
inference.initialize(n_print=10, step_size=0.15)

tf.global_variables_initializer().run()

for i in range(inference.n_iter):

    info_dict = inference.update()
    inference.print_progress(info_dict)
    
inference.finalize()

10000/10000 [100%] ██████████████████████████████ Elapsed: 32s | Acceptance Rate: 0.416

Evaluation

In [33]:
w_mask_samples = w_mask.sample(sample_shape=P).eval()
w_raw_samples = qw.sample(sample_shape=P).eval()
w_samples = w_mask_samples * w_raw_samples
In [34]:
g = sns.JointGrid(*w_samples.T,
                  xlim=(w1_min, w1_max),
                  ylim=(w2_min, w2_max))
g = g.plot_joint(plt.scatter, s=3.**2, alpha=.4)
g = g.plot_marginals(sns.distplot, kde=True)
g = g.set_axis_labels('$w_1$', '$w_2$')
In [35]:
mixtures = np.right_shift(np.packbits(w_mask_samples, 
                                      axis=1), 6).flatten()
In [36]:
fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(9, 4))

fig.suptitle('Posterior samples')

ax1.scatter(*w_samples.T, s=4.**2, alpha=.6)
ax2.scatter(*w_samples.T, c=mixtures, s=4.**2, alpha=.6, cmap='Blues')

ax1.set_title('Zoomed out')

ax1.set_xlabel('$w_1$')
ax1.set_ylabel('$w_2$')

ax2.set_title('Colored by mixture')

ax2.set_xlabel('$w_1$')

plt.show()

Alternative definition of logistic regression model as multi-input and multi-output model in Keras

Comments

In [1]:
%matplotlib inline
In [2]:
import numpy as np

import matplotlib.pyplot as plt

import keras.backend as K

from keras.layers import Input, Dense
from keras.models import Sequential, Model

from sklearn.model_selection import train_test_split
from sklearn.datasets import make_blobs

Using TensorFlow backend.
/Users/tiao/.virtualenvs/anmoku/lib/python3.6/importlib/_bootstrap.py:205: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6
  return f(*args, **kwds)
In [3]:
'TensorFlow version: ' + K.tf.__version__
Out[3]:
'TensorFlow version: 1.4.0'

Constants

In [4]:
n_samples = 100
n_features = 2
n_classes = 2
seed = 42
rng = np.random.RandomState(seed)

Toy Dataset ("Gaussian blobs")

In [5]:
x_test, y_test = make_blobs(n_samples=n_samples, centers=n_classes, random_state=rng)
In [6]:
# class labels are balanced
np.sum(y_test)
Out[6]:
50
In [7]:
fig, ax = plt.subplots(figsize=(7, 5))

cb = ax.scatter(*x_test.T, c=y_test, cmap='coolwarm')
fig.colorbar(cb, ax=ax)

ax.set_xlabel('$x_1$')
ax.set_ylabel('$x_2$')

plt.show()

Typical Model Specification for Logistic Regression

In [8]:
classifier = Sequential([
    Dense(16, input_dim=n_features, activation='relu'),
    Dense(32, activation='relu'),
    Dense(32, activation='relu'),
    Dense(1, activation='sigmoid')
])
classifier.compile(optimizer='rmsprop', loss='binary_crossentropy')
In [9]:
loss = classifier.evaluate(x_test, y_test)
loss

100/100 [==============================] - 0s 1ms/step
Out[9]:
0.66595284819602962

Alternative Specification

An alternative specification isolates the positive and negative samples and explicitly requires an input for each class. The loss that is optimized is then the sum of the binary cross-entropy losses for each class. Since we fix the corresponding target labels to be all ones or zeros accordingly, the binary cross-entropy losses for each class both result in one of the complementary terms, and averaging them would result in the usual binary cross-entropy loss on all samples with their corresponding labels.

This trick is crucial for many model specifications in keras-adversarial.

In [10]:
pos = Input(shape=(n_features,))
neg = Input(shape=(n_features,))

# make use of the classifier defined earlier
y_pred_pos = classifier(pos)
y_pred_neg = classifier(neg)

# define a multi-input, multi-output model
classifier_alt = Model([pos, neg], [y_pred_pos, y_pred_neg])
classifier_alt.compile(optimizer='rmsprop', loss='binary_crossentropy')
In [11]:
losses = classifier_alt.evaluate(
    [
        x_test[y_test == 1], 
        x_test[y_test == 0]
    ], 
    [
        np.ones(n_samples // 2), 
        np.zeros(n_samples // 2)
    ]
)
losses

50/50 [==============================] - 0s 1ms/step
Out[11]:
[1.3319057083129884, 1.0514429426193237, 0.28046274423599243]

The loss that actually gets optimized is the first value above, the sum of the subsequent values. The mean of them is the required binary cross-entropy loss.

In [12]:
.5 * losses[0]
Out[12]:
0.66595285415649419
In [13]:
# alternatively
np.mean(losses[1:])
Out[13]:
0.66595284342765804
In [14]:
np.allclose(loss, np.mean(losses[1:]))
Out[14]:
True

Difference between Keras stack and concatenate

Comments

In [1]:
import keras.backend as K

Using TensorFlow backend.
In [2]:
'TensorFlow version: ' + K.tf.__version__
Out[2]:
'TensorFlow version: 1.4.0'
Concatenate
In [3]:
K.eval(K.concatenate([K.random_uniform(shape=(3, 4)),
                      K.random_uniform(shape=(3, 4))], axis=0)).shape
Out[3]:
(6, 4)
In [4]:
K.eval(K.concatenate([K.random_uniform(shape=(3, 4)),
                      K.random_uniform(shape=(3, 4))], axis=1)).shape
Out[4]:
(3, 8)
Stack
In [5]:
K.eval(K.stack([K.random_uniform(shape=(3, 4)),
                K.random_uniform(shape=(3, 4))], axis=0)).shape
Out[5]:
(2, 3, 4)
In [6]:
K.eval(K.stack([K.random_uniform(shape=(3, 4)),
                K.random_uniform(shape=(3, 4))], axis=1)).shape
Out[6]:
(3, 2, 4)
In [7]:
K.eval(K.stack([K.random_uniform(shape=(3, 4)),
                K.random_uniform(shape=(3, 4))], axis=2)).shape
Out[7]:
(3, 4, 2)

Keras Constant Input Layers with Fixed Source of Stochasticity

Comments

In [1]:
%matplotlib notebook
In [2]:
import numpy as np
import keras.backend as K

from keras.layers import Input, Activation, Add, GaussianNoise
from keras.models import Sequential, Model

Using TensorFlow backend.

Rationale

TODO

In [3]:
random_tensor = K.random_normal(shape=(8, 3), seed=42)
In [4]:
K.eval(random_tensor)
Out[4]:
array([[-0.28077507, -0.13775212, -0.67632961],
       [ 0.02458041, -0.89358455, -0.82847327],
       [ 1.2068944 ,  1.38101566, -1.45579767],
       [-0.24621388, -1.36084056,  1.08796036],
       [-0.35116589, -0.51385337,  3.41172075],
       [ 0.05885483,  0.89180237, -0.7528832 ],
       [-0.4335728 ,  2.45385313,  0.31374422],
       [-0.52736205,  0.85249925, -0.5379132 ]], dtype=float32)
In [5]:
K.eval(random_tensor)
Out[5]:
array([[ 0.36944136, -0.06497762,  1.05423534],
       [ 0.92629176,  0.45142221,  0.6538806 ],
       [ 0.00987345, -0.75727743,  1.19744813],
       [ 0.10721783, -1.34733653,  0.69856125],
       [ 1.34215105,  0.19264366, -0.02015864],
       [ 0.61278504,  0.43748191,  1.21581125],
       [ 0.42827308, -1.2276696 , -2.39826727],
       [-0.21679108,  0.05826041,  0.10147382]], dtype=float32)
In [6]:
x = Input(shape=(784,))
eps = Input(tensor=K.random_normal(shape=(K.shape(x)[0], 3), seed=42))
In [7]:
try:
    m = Model(x, eps)
except RuntimeError as e:
    print(e)

Graph disconnected: cannot obtain value for tensor Tensor("random_normal_1:0", shape=(?, 3), dtype=float32) at layer "input_2". The following previous layers were accessed without issue: []
In [8]:
m = Model([x, eps], eps)
In [9]:
m.predict(np.ones((8, 784)))
Out[9]:
array([[-0.28077507, -0.13775212, -0.67632961],
       [ 0.02458041, -0.89358455, -0.82847327],
       [ 1.2068944 ,  1.38101566, -1.45579767],
       [-0.24621388, -1.36084056,  1.08796036],
       [-0.35116589, -0.51385337,  3.41172075],
       [ 0.05885483,  0.89180237, -0.7528832 ],
       [-0.4335728 ,  2.45385313,  0.31374422],
       [-0.52736205,  0.85249925, -0.5379132 ]], dtype=float32)
In [10]:
m.predict([np.ones((8, 784))])
Out[10]:
array([[ 0.36944136, -0.06497762,  1.05423534],
       [ 0.92629176,  0.45142221,  0.6538806 ],
       [ 0.00987345, -0.75727743,  1.19744813],
       [ 0.10721783, -1.34733653,  0.69856125],
       [ 1.34215105,  0.19264366, -0.02015864],
       [ 0.61278504,  0.43748191,  1.21581125],
       [ 0.42827308, -1.2276696 , -2.39826727],
       [-0.21679108,  0.05826041,  0.10147382]], dtype=float32)
In [11]:
try:
    m.predict([np.ones((8, 784)), np.ones((8, 3))])
except ValueError as e:
    print(e)

Error when checking model : the list of Numpy arrays that you are passing to your model is not the size the model expected. Expected to see 1 array(s), but instead got the following list of 2 arrays: [array([[ 1.,  1.,  1., ...,  1.,  1.,  1.],
       [ 1.,  1.,  1., ...,  1.,  1.,  1.],
       [ 1.,  1.,  1., ...,  1.,  1.,  1.],
       ..., 
       [ 1.,  1.,  1., ...,  1.,  1.,  1.],
       [ 1...

Working with Pandas MultiIndex Dataframes: Reading and Writing to CSV and HDF5

Comments

In [1]:
%matplotlib notebook
In [2]:
import numpy as np
import pandas as pd

import matplotlib.pyplot as plt

Rationale

For some certain loss functions, such the the negative evidence lower bound (NELBO) in variational inference, they are generally analytically intractable and thus unavailable in closed-form. As such, we might need to resort to taking stochastic estimates of the loss function. In these situations, it is very important to study and understand the robustness of the estimations we are making, particularly in terms of bias and variance. When proposing a new estimator, we may be interested in evaluating the loss at a fined-grained level - not only per batch, but perhaps even per data-point.

This notebook explores storing the recorded losses in Pandas Dataframes. The recorded losses are 3d, with dimensions corresponding to epochs, batches, and data-points. Specifically, they are of shape (n_epochs, n_batches, batch_size). Instead of using the deprecated Panel functionality from Pandas, we explore the preferred MultiIndex Dataframe.

Lastly, we play around with various data serialization formats supported out-of-the-box by Pandas. This might be useful if the training is GPU-intensive, so the script runs and records the loss remotely on a supercomputer, and we must write the results to file, download them and finally analyze them locally. This is usually trivial, but it is unclear what the behaviour is for more complex MultiIndex dataframes. We restrict our attention to the CSV format, which is human-friendly but very slow and inefficient, and the HDF5, which is basically diametrically opposed - it's basically completely inscrutable, but is very fast and takes up laess space.

Synthetic Data

In [3]:
# create some noise
a = np.random.randn(50, 600, 100)
a.shape
Out[3]:
(50, 600, 100)
In [4]:
# create some noise with higher variance and add bias.
b = 2. * np.random.randn(*a.shape) + 1.
b.shape
Out[4]:
(50, 600, 100)
In [5]:
# manufacture some loss function
# there are n_epochs * n_batchs * batch_size 
# recorded values of the loss
loss = 10 / np.linspace(1, 100, a.size)
loss.shape
Out[5]:
(3000000,)

MultiIndex Dataframe

In [6]:
# we will create the indices from the 
# product of these iterators
list(map(range, a.shape))
Out[6]:
[range(0, 50), range(0, 600), range(0, 100)]
In [7]:
# create the MultiIndex
index = pd.MultiIndex.from_product(
    list(map(range, a.shape)), 
    names=['epoch', 'batch', 'datapoint']
)
In [8]:
# create the dataframe that records the two losses
df = pd.DataFrame(
    dict(loss1=loss+np.ravel(a), 
         loss2=loss+np.ravel(b)), 
    index=index
)
df
Out[8]:
loss1 loss2
epoch batch datapoint
0 0 0 10.837250 10.228649
1 9.383650 9.601012
2 9.102928 12.792865
3 9.149701 11.307185
4 9.181607 9.905578
5 8.984361 11.646015
6 8.935352 10.793933
7 9.273609 9.421425
8 10.846009 9.916008
9 10.288851 7.250876
10 10.360709 10.911360
11 9.514765 7.339939
12 9.922280 10.494360
13 9.094041 13.302492
14 9.693384 12.187093
15 9.675839 13.418631
16 11.502391 9.470244
17 10.958843 12.709454
18 10.819225 11.700684
19 9.412562 10.272870
20 10.424428 10.477799
21 9.009290 13.920663
22 8.127529 13.179637
23 8.939673 13.091603
24 8.064599 7.311483
25 9.924553 15.597797
26 8.572734 14.338683
27 10.294223 7.761236
28 11.191123 8.993673
29 9.424269 12.067344
... ... ... ... ...
49 599 70 1.975951 2.746047
71 1.169242 4.194400
72 0.248216 0.730370
73 -0.271330 0.415903
74 -0.888580 0.208060
75 -0.624063 1.081515
76 -1.422904 0.398015
77 0.332523 1.892470
78 -0.224471 -2.839332
79 0.405337 0.266035
80 0.223712 5.810484
81 -0.508689 7.535930
82 -1.915472 -0.275332
83 -0.597498 2.799929
84 -0.443378 2.202897
85 0.610826 1.825950
86 0.305465 0.757416
87 -1.139339 3.221787
88 -1.893639 0.520711
89 -0.286300 3.112420
90 1.268100 1.341298
91 0.251563 1.040859
92 0.083156 1.311108
93 -0.554107 8.272526
94 -2.415105 2.607663
95 0.335266 2.038404
96 0.554412 2.200551
97 0.392182 1.444542
98 -0.252059 1.641488
99 -0.070091 1.490349

3000000 rows × 2 columns

Visualization

In this contrived scenario, loss2 is more biased and has higher variance.

In [9]:
# some basic plotting
fig, ax = plt.subplots()

df.groupby(['epoch', 'batch']).mean().plot(ax=ax)

plt.show()

CSV Read/Write

In [10]:
%%time

df.to_csv('losses.csv')

CPU times: user 9.56 s, sys: 184 ms, total: 9.74 s
Wall time: 13.3 s
In [11]:
!ls -lh losses.csv

-rwxrwxrwx 1 tiao tiao 138M Nov  8 03:14 losses.csv
In [12]:
%%time

df_from_csv = pd.read_csv('losses.csv', index_col=['epoch', 'batch', 'datapoint'], float_precision='high')

/home/tiao/.virtualenvs/anmoku/lib/python3.5/site-packages/numpy/lib/arraysetops.py:463: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison
  mask |= (ar1 == a)

CPU times: user 1.47 s, sys: 108 ms, total: 1.58 s
Wall time: 3.73 s
In [13]:
# does not recover exactly due to insufficient floating point precision
df_from_csv.equals(df)
Out[13]:
False
In [14]:
# but it has recovered it up to some tiny epsilon
((df-df_from_csv)**2 < 1e-25).all()
Out[14]:
loss1    True
loss2    True
dtype: bool

HDF5 Read/Write

HDF5 writing is orders of magnitude faster.

In [15]:
%%time

df.to_hdf('store.h5', key='losses')

CPU times: user 44 ms, sys: 72 ms, total: 116 ms
Wall time: 720 ms

Furthermore, the file sizes are significantly smaller.

In [16]:
!ls -lh store.h5

-rwxrwxrwx 1 tiao tiao 58M Nov  8 03:15 store.h5
In [17]:
%%time

df_from_hdf = pd.read_hdf('store.h5', key='losses')

CPU times: user 28 ms, sys: 28 ms, total: 56 ms
Wall time: 105 ms

Lastly, it is far more numerical precise.

In [18]:
df.equals(df_from_hdf)
Out[18]:
True

Working with Samples of Distributions over Convolutional Kernels

Comments

In [1]:
%matplotlib inline
%config InlineBackend.figure_format = 'svg'
In [2]:
import numpy as np
import tensorflow as tf

import matplotlib.pyplot as plt

from tensorflow.examples.tutorials.mnist import input_data as mnist_data
In [3]:
tf.__version__
Out[3]:
'1.2.1'
In [4]:
sess = tf.InteractiveSession()
In [5]:
mnist = mnist_data.read_data_sets("/home/tiao/Desktop/MNIST")

Extracting /home/tiao/Desktop/MNIST/train-images-idx3-ubyte.gz
Extracting /home/tiao/Desktop/MNIST/train-labels-idx1-ubyte.gz
Extracting /home/tiao/Desktop/MNIST/t10k-images-idx3-ubyte.gz
Extracting /home/tiao/Desktop/MNIST/t10k-labels-idx1-ubyte.gz
In [6]:
# 50 single-channel (grayscale) 28x28 images
x = mnist.train.images[:50].reshape(-1, 28, 28, 1)
x.shape
Out[6]:
(50, 28, 28, 1)
In [7]:
fig, ax = plt.subplots(figsize=(5, 5))

# showing an arbitrarily chosen image
ax.imshow(np.squeeze(x[5], axis=-1), cmap='gray')

plt.show()

Standard 2D Convolution with conv2d

In [8]:
# 32 kernels of size 5x5x1
kernel = tf.truncated_normal([5, 5, 1, 32], stddev=0.1)
kernel.get_shape().as_list()
Out[8]:
[5, 5, 1, 32]
In [9]:
x_conved = tf.nn.conv2d(x, kernel, 
                        strides=[1, 1, 1, 1], 
                        padding='SAME')
x_conved.get_shape().as_list()
Out[9]:
[50, 28, 28, 32]
In [10]:
x_conved[5, ..., 0].eval().shape
Out[10]:
(28, 28)
In [11]:
fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(9, 4))

# showing what the 0th filter looks like
ax1.imshow(kernel[..., 0, 0].eval(), cmap='gray')

# show the previous arbitrarily chosen image
# convolved with the 0th filter
ax2.imshow(x_conved[5, ..., 0].eval(), cmap='gray')

plt.show()

Sample from a Distribution over Kernels

In [12]:
# 8x32 kernels of size 5x5x1
kernels = tf.truncated_normal([8, 5, 5, 1, 32], stddev=0.1)
kernels.get_shape().as_list()
Out[12]:
[8, 5, 5, 1, 32]

Approach 1: Map over samples with conv2d

In [13]:
x_tiled = tf.tile(tf.expand_dims(x, 0), [8, 1, 1, 1, 1])
x_tiled.get_shape().as_list()
Out[13]:
[8, 50, 28, 28, 1]
In [19]:
tf.nn.conv2d(x_tiled[0], kernels[0], 
             strides=[1, 1, 1, 1], 
             padding='SAME').get_shape().as_list()
Out[19]:
[50, 28, 28, 32]
In [15]:
x_conved1 = tf.map_fn(lambda args: tf.nn.conv2d(*args, strides=[1, 1, 1, 1], padding='SAME'),
                      elems=(x_tiled, kernels), dtype=tf.float32)
x_conved1.get_shape().as_list()
Out[15]:
[8, 50, 28, 28, 32]

Approach 2: Flattening

In [16]:
kernels_flat = tf.reshape(tf.transpose(kernels, 
                                       perm=(1, 2, 3, 4, 0)), 
                          shape=(5, 5, 1, 32*8))
kernels_flat.get_shape().as_list()
Out[16]:
[5, 5, 1, 256]
In [17]:
x_conved2 = tf.transpose(tf.reshape(tf.nn.conv2d(x, kernels_flat, 
                                                 strides=[1, 1, 1, 1], 
                                                 padding='SAME'), 
                                    shape=(50, 28, 28, 32, 8)), 
                         perm=(4, 0, 1, 2, 3))
x_conved2.get_shape().as_list()
Out[17]:
[8, 50, 28, 28, 32]
In [18]:
tf.reduce_all(tf.equal(x_conved1, x_conved2)).eval()
Out[18]:
True

Variational Inference with Implicit Approximate Inference Models - @fhuszar's Explaining Away Example Pt. 1 (WIP)

Comments

In [1]:
%matplotlib inline
%config InlineBackend.figure_format = 'svg'
In [70]:
import numpy as np
import keras.backend as K

import matplotlib.pyplot as plt
import seaborn as sns

from scipy.stats import logistic, multivariate_normal, norm
from scipy.special import expit

from keras.models import Model, Sequential
from keras.layers import Activation, Add, Dense, Dot, Input
from keras.optimizers import Adam
from keras.utils.vis_utils import model_to_dot

from mpl_toolkits.mplot3d import Axes3D
from matplotlib.animation import FuncAnimation

from IPython.display import HTML, SVG, display_html
from tqdm import tnrange, tqdm_notebook
In [3]:
# display animation inline
plt.rc('animation', html='html5')
plt.style.use('seaborn-notebook')
sns.set_context('notebook')
In [4]:
np.set_printoptions(precision=2,
                    edgeitems=3,
                    linewidth=80,
                    suppress=True)
In [5]:
K.tf.__version__
Out[5]:
'1.2.1'
In [6]:
LATENT_DIM = 2
NOISE_DIM = 3
BATCH_SIZE = 200
PRIOR_VARIANCE = 2.
LEARNING_RATE = 3e-3
PRETRAIN_EPOCHS = 60

Bayesian Logistic Regression (Synthetic Data)

In [7]:
z_min, z_max = -5, 5
In [8]:
z1, z2 = np.mgrid[z_min:z_max:300j, z_min:z_max:300j]
In [9]:
z_grid = np.dstack((z1, z2))
z_grid.shape
Out[9]:
(300, 300, 2)
In [10]:
prior = multivariate_normal(mean=np.zeros(LATENT_DIM), 
                            cov=PRIOR_VARIANCE)
In [11]:
log_prior = prior.logpdf(z_grid)
log_prior.shape
Out[11]:
(300, 300)
In [13]:
np.allclose(log_prior, 
            -.5*np.sum(z_grid**2, axis=2)/PRIOR_VARIANCE \
            -np.log(2*np.pi*PRIOR_VARIANCE))
Out[13]:
True
In [15]:
fig, ax = plt.subplots(figsize=(5, 5))

ax.contourf(z1, z2, log_prior, cmap='magma')

ax.set_xlabel('$w_1$')
ax.set_ylabel('$w_2$')

ax.set_xlim(z_min, z_max)
ax.set_ylim(z_min, z_max)

plt.show()
In [16]:
x = np.array([0, 5, 8, 12, 50])
In [37]:
def log_likelihood(z, x, beta_0=3., beta_1=1.):
    beta = beta_0 + np.sum(beta_1*np.maximum(0, z**3), axis=-1)
    return -np.log(beta) - x/beta
In [44]:
llhs = log_likelihood(z_grid, x.reshape(-1, 1, 1))
llhs.shape
Out[44]:
(5, 300, 300)
In [59]:
fig, axes = plt.subplots(ncols=len(x), nrows=1, figsize=(20, 4))
fig.tight_layout()

for i, ax in enumerate(axes):
    
    ax.contourf(z1, z2, llhs[i,::,::], cmap=plt.cm.magma)

    ax.set_xlim(z_min, z_max)
    ax.set_ylim(z_min, z_max)
    
    ax.set_title('$p(x = {{{0}}} \mid z)$'.format(x[i]))
    ax.set_xlabel('$z_1$')    
    
    if not i:
        ax.set_ylabel('$z_2$')

plt.show()
In [60]:
fig, ax = plt.subplots(figsize=(5, 5))

ax.contourf(z1, z2, np.sum(llhs, axis=0), 
                cmap=plt.cm.magma)

ax.set_xlabel('$z_1$')
ax.set_ylabel('$z_2$')

ax.set_xlim(z_min, z_max)
ax.set_ylim(z_min, z_max)

plt.show()
In [61]:
fig, axes = plt.subplots(ncols=len(x), nrows=1, figsize=(20, 4))
fig.tight_layout()

for i, ax in enumerate(axes):
    
    ax.contourf(z1, z2,  np.exp(log_prior+llhs[i,::,::]), 
                cmap='magma')

    ax.set_xlim(z_min, z_max)
    ax.set_ylim(z_min, z_max)
    
    ax.set_title('$Zp(z \mid x = {{{0}}})$'.format(x[i]))
    ax.set_xlabel('$z_1$')    
    
    if not i:
        ax.set_ylabel('$z_2$')

plt.show()
In [63]:
fig, ax = plt.subplots(figsize=(5, 5))

ax.contourf(z1, z2, 
            np.exp(log_prior+np.sum(llhs, axis=0)), 
            cmap='magma')

ax.set_xlabel('$z_1$')
ax.set_ylabel('$z_2$')

ax.set_xlim(z_min, z_max)
ax.set_ylim(z_min, z_max)

plt.show()

Model Definitions

Density Ratio Estimator (Discriminator) Model

$T_{\psi}(x, z)$

In [84]:
x_input = Input(shape=(1,), name='x')
x_hidden = Dense(10, activation='relu')(x_input)
x_hidden = Dense(20, activation='relu')(x_hidden)
In [85]:
z_input = Input(shape=(LATENT_DIM,), name='z')
z_hidden = Dense(10, activation='relu')(z_input)
z_hidden = Dense(20, activation='relu')(z_hidden)
In [86]:
discrim_hidden = Add()([x_hidden, z_hidden])
discrim_hidden = Dense(10, activation='relu')(discrim_hidden)
discrim_hidden = Dense(20, activation='relu')(discrim_hidden)
discrim_logit = Dense(1, activation=None, 
                       name='logit')(discrim_hidden)
discrim_out = Activation('sigmoid')(discrim_logit)
In [87]:
discriminator = Model(inputs=[x_input, z_input], outputs=discrim_out)
discriminator.compile(optimizer=Adam(lr=LEARNING_RATE),
                      loss='binary_crossentropy',
                      metrics=['binary_accuracy'])
In [88]:
ratio_estimator = Model(
    inputs=discriminator.inputs, 
    outputs=discrim_logit)
In [89]:
SVG(model_to_dot(discriminator, show_shapes=True)
    .create(prog='dot', format='svg'))
Out[89]:
G 140115456278480 x: InputLayerinput:output:(None, 1)(None, 1)140115456277696 dense_17: Denseinput:output:(None, 1)(None, 10)140115456278480->140115456277696 140115452033176 z: InputLayerinput:output:(None, 2)(None, 2)140115453473064 dense_19: Denseinput:output:(None, 2)(None, 10)140115452033176->140115453473064 140115453258048 dense_18: Denseinput:output:(None, 10)(None, 20)140115456277696->140115453258048 140115452043616 dense_20: Denseinput:output:(None, 10)(None, 20)140115453473064->140115452043616 140115531775392 add_8: Addinput:output:[(None, 20), (None, 20)](None, 20)140115453258048->140115531775392 140115452043616->140115531775392 140115531775168 dense_21: Denseinput:output:(None, 20)(None, 10)140115531775392->140115531775168 140115531775504 dense_22: Denseinput:output:(None, 10)(None, 20)140115531775168->140115531775504 140115543993984 logit: Denseinput:output:(None, 20)(None, 1)140115531775504->140115543993984 140115453098640 activation_2: Activationinput:output:(None, 1)(None, 1)140115543993984->140115453098640
In [105]:
np.ones((32, 5)) + np.ones((16, 5))

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-105-a367db8def8f> in <module>()
----> 1 np.ones((32, 5)) + np.ones((16, 5))

ValueError: operands could not be broadcast together with shapes (32,5) (16,5)
In [112]:
z_grid_ratio = ratio_estimator.predict([np.ones((16, 1)), np.ones((32, 2))])
z_grid_ratio.shape
Out[112]:
(16, 1)

Initial density ratio, prior to any training

In [ ]:
fig, ax = plt.subplots(figsize=(5, 5))

ax.contourf(w1, w2, w_grid_ratio, cmap=plt.cm.magma)

ax.set_xlabel('$w_1$')
ax.set_ylabel('$w_2$')

ax.set_xlim(w_min, w_max)
ax.set_ylim(w_min, w_max)

plt.show()
In [ ]:
discriminator.evaluate(prior.rvs(size=5), np.zeros(5))

Approximate Inference Model

$z_{\phi}(x, \epsilon)$

Here we only consider

$z_{\phi}(\epsilon)$

$z_{\phi}: \mathbb{R}^3 \to \mathbb{R}^2$

In [ ]:
inference = Sequential()
inference.add(Dense(10, input_dim=NOISE_DIM, activation='relu'))
inference.add(Dense(20, activation='relu'))
inference.add(Dense(LATENT_DIM, activation=None))
inference.summary()

The variational parameters $\phi$ are the trainable weights of the approximate inference model

In [ ]:
phi = inference.trainable_weights
phi
In [ ]:
SVG(model_to_dot(inference, show_shapes=True)
    .create(prog='dot', format='svg'))
In [ ]:
w_sample_prior = prior.rvs(size=BATCH_SIZE)
w_sample_prior.shape
In [ ]:
eps = np.random.randn(BATCH_SIZE, NOISE_DIM)
w_sample_posterior = inference.predict(eps)
w_sample_posterior.shape
In [ ]:
inputs = np.vstack((w_sample_prior, w_sample_posterior))
targets = np.hstack((np.zeros(BATCH_SIZE), np.ones(BATCH_SIZE)))
In [ ]:
fig, ax = plt.subplots(figsize=(5, 5))

ax.contourf(w1, w2, 
            np.exp(log_prior+np.sum(llhs, axis=2)), 
            cmap=plt.cm.magma)

ax.scatter(*inputs.T, c=targets, s=4.**2, alpha=.8, cmap='coolwarm')

ax.set_xlabel('$w_1$')
ax.set_ylabel('$w_2$')

ax.set_xlim(w_min, w_max)
ax.set_ylim(w_min, w_max)

plt.show()
In [ ]:
metrics = discriminator.evaluate(inputs, targets)
In [ ]:
w_grid_ratio = ratio_estimator.predict(w_grid.reshape(300*300, 2))
w_grid_ratio = w_grid_ratio.reshape(300, 300)
In [ ]:
metrics
In [ ]:
metrics_dict = dict(zip(discriminator.metrics_names, metrics))
In [ ]:
fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize=(9, 4))

metrics_plots = {k:ax1.plot([], label=k)[0] 
                 for k in ['loss']} # discriminator.metrics_names}

ax1.set_xlabel('epoch')
ax1.legend(loc='upper left')

ax2.contourf(w1, w2, w_grid_ratio, cmap='magma')
ax2.scatter(*inputs.T, c=targets, s=4.**2, alpha=.8, cmap='coolwarm')

ax2.set_xlabel('$w_1$')
ax2.set_ylabel('$w_2$')

ax2.set_xlim(w_min, w_max)
ax2.set_ylim(w_min, w_max)

plt.show()
Discriminator pre-training
In [ ]:
def train_animate(epoch_num, prog_bar, batch_size=200, steps_per_epoch=15):

    # Single training epoch
    
    for step in tnrange(steps_per_epoch, unit='step', leave=False):

        w_sample_prior = prior.rvs(size=batch_size)

        eps = np.random.randn(batch_size, NOISE_DIM)
        w_sample_posterior = inference.predict(eps)

        inputs = np.vstack((w_sample_prior, w_sample_posterior))
        targets = np.hstack((np.zeros(batch_size), np.ones(batch_size)))

        metrics = discriminator.train_on_batch(inputs, targets)

    # Plot Metrics
        
    metrics_dict = dict(zip(discriminator.metrics_names, metrics))

    for metric in metrics_plots:
        metrics_plots[metric].set_xdata(np.append(metrics_plots[metric].get_xdata(), 
                                                  epoch_num))    
        metrics_plots[metric].set_ydata(np.append(metrics_plots[metric].get_ydata(), 
                                                  metrics_dict[metric]))
        metrics_plots[metric].set_label('{} ({:.2f})' \
                                        .format(metric, 
                                                metrics_dict[metric]))
    
    ax1.set_xlabel('epoch {:2d}'.format(epoch_num))
    ax1.legend(loc='upper left')

    ax1.relim()
    ax1.autoscale_view()
    
    # Contour Plot
    
    ax2.cla()

    w_grid_ratio = ratio_estimator.predict(w_grid.reshape(300*300, 2))
    w_grid_ratio = w_grid_ratio.reshape(300, 300)

    ax2.contourf(w1, w2, w_grid_ratio, cmap='magma')
    ax2.scatter(*inputs.T, c=targets, s=4.**2, alpha=.8, cmap='coolwarm')

    ax2.set_xlabel('$w_1$')
    ax2.set_ylabel('$w_2$')

    ax2.set_xlim(w_min, w_max)
    ax2.set_ylim(w_min, w_max)
    
    # Progress Bar Updates
    
    prog_bar.update()
    prog_bar.set_postfix(**metrics_dict)

    return list(metrics_plots.values())
In [ ]:
# main training loop is managed by higher-order
# FuncAnimation which makes calls to an `animate` 
# function that encapsulates the logic of single
# training epoch. Has benefit of producing 
# animation but can incur significant overhead
with tqdm_notebook(total=PRETRAIN_EPOCHS, 
                   unit='epoch', leave=True) as prog_bar:

    anim = FuncAnimation(fig, 
                         train_animate,
                         fargs=(prog_bar,),
                         frames=PRETRAIN_EPOCHS,
                         interval=200, # 5 fps
                         blit=True)

    anim_html5_video = anim.to_html5_video()
In [ ]:
HTML(anim_html5_video)
In [ ]:
inputs = np.vstack((w_sample_prior, w_sample_posterior))
targets = np.hstack((np.zeros(BATCH_SIZE), np.ones(BATCH_SIZE)))
In [ ]:
metrics = discriminator.evaluate(inputs, targets)
In [ ]:
w_grid_ratio = ratio_estimator.predict(w_grid.reshape(300*300, 2))
w_grid_ratio = w_grid_ratio.reshape(300, 300)
In [ ]:
fig, ax = plt.subplots(figsize=(5, 5))

ax.contourf(w1, w2, w_grid_ratio, cmap='magma')

ax.scatter(*inputs.T, c=targets, s=4.**2, alpha=.8, cmap='coolwarm')

metrics_dict = dict(zip(discriminator.metrics_names, metrics))

props = dict(boxstyle='round', facecolor='w', alpha=0.5)

ax.text(0.05, 0.05, 
        ('accuracy: {binary_accuracy:.2f}\n'        
         'loss: {loss:.2f}').format(**metrics_dict), 
        transform=ax.transAxes, bbox=props)

ax.set_xlabel('$w_1$')
ax.set_ylabel('$w_2$')

ax.set_xlim(w_min, w_max)
ax.set_ylim(w_min, w_max)

plt.show()

Evidence lower bound

In [ ]:
def set_trainable(model, trainable):
    """inorder traversal"""
    model.trainable = trainable

    if isinstance(model, Model): # i.e. has layers
        for layer in model.layers:
            set_trainable(layer, trainable)
In [ ]:
y_pred = K.sigmoid(K.dot(
    K.constant(w_grid),
    K.transpose(K.constant(X))))
y_pred
In [ ]:
y_true = K.ones((300, 300, 1))*K.constant(y)
y_true
In [ ]:
llhs_keras = - K.binary_crossentropy(
                   y_pred, 
                   y_true, 
                   from_logits=False)
In [ ]:
sess = K.get_session()
In [ ]:
np.allclose(np.sum(llhs, axis=-1),
            sess.run(K.sum(llhs_keras, axis=-1)))
In [ ]:
fig, ax = plt.subplots(figsize=(5, 5))

ax.contourf(w1, w2, sess.run(K.sum(llhs_keras, axis=-1)), 
            cmap=plt.cm.magma)

ax.set_xlabel('$w_1$')
ax.set_ylabel('$w_2$')

ax.set_xlim(w_min, w_max)
ax.set_ylim(w_min, w_max)

plt.show()

Reweight likelihood term!

In [ ]:
def make_elbo(ratio_estimator):
    
    set_trainable(ratio_estimator, False)
    
    def elbo(y_true, w_sample):
        kl_estimate = ratio_estimator(w_sample)
        y_pred = K.dot(w_sample, K.transpose(K.constant(X)))
        log_likelihood = - K.binary_crossentropy(y_pred, y_true, 
                                                 from_logits=True)
        return K.mean(2.*log_likelihood-kl_estimate, axis=-1)

    return elbo
In [ ]:
elbo = make_elbo(ratio_estimator)
In [ ]:
fig, ax = plt.subplots(figsize=(5, 5))

ax.contourf(w1, w2, sess.run(elbo(y_true, K.constant(w_grid))), 
            cmap=plt.cm.magma)

ax.set_xlabel('$w_1$')
ax.set_ylabel('$w_2$')

ax.set_xlim(w_min, w_max)
ax.set_ylim(w_min, w_max)

plt.show()
In [ ]:
inference_loss = lambda y_true, w_sample: -make_elbo(ratio_estimator)(y_true, w_sample)
In [ ]:
inference.compile(loss=inference_loss, 
                  optimizer=Adam(lr=LEARNING_RATE))
In [ ]:
eps = np.random.randn(BATCH_SIZE, NOISE_DIM)
In [ ]:
y_true = K.repeat_elements(K.expand_dims(K.constant(y), axis=0), 
                           axis=0, rep=BATCH_SIZE)
y_true
In [ ]:
sess.run(K.mean(elbo(y_true, inference(K.constant(eps))), axis=-1))
In [ ]:
inference.evaluate(eps, np.tile(y, reps=(BATCH_SIZE, 1)))

Adversarial Training

In [ ]:
fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize=(9, 4))

global_epoch = 0

loss_plot_inference, = ax1.plot([], label='inference')
loss_plot_discrim, = ax1.plot([], label='discriminator')

ax1.set_xlabel('epoch')
ax1.set_ylabel('loss')
ax1.legend(loc='upper left')

ax2.contourf(w1, w2, w_grid_ratio, cmap='magma')
ax2.scatter(*inputs.T, c=targets, s=4.**2, alpha=.8, cmap='coolwarm')

ax2.set_xlabel('$w_1$')
ax2.set_ylabel('$w_2$')

ax2.set_xlim(w_min, w_max)
ax2.set_ylim(w_min, w_max)

plt.show()
In [ ]:
def train_animate(epoch_num, prog_bar, batch_size=200, 
                  steps_per_epoch=15):

    global global_epoch, loss_plot_inference, loss_plot_discrim
    
    # Single training epoch

    ## Ratio estimator training
        
    set_trainable(discriminator, True)

    for _ in tnrange(3*50, unit='step', desc='discriminator', 
                     leave=False):

        w_sample_prior = prior.rvs(size=BATCH_SIZE)

        eps = np.random.randn(BATCH_SIZE, NOISE_DIM)
        w_sample_posterior = inference.predict(eps)

        inputs = np.vstack((w_sample_prior, w_sample_posterior))
        targets = np.hstack((np.zeros(BATCH_SIZE), np.ones(BATCH_SIZE)))

        metrics_discrim = discriminator.train_on_batch(inputs, targets)

    metrics_dict_discrim = dict(zip(discriminator.metrics_names, 
                                    np.atleast_1d(metrics_discrim)))
    
    ## Inference model training
    
    set_trainable(ratio_estimator, False)

    y_tiled = np.tile(y, reps=(BATCH_SIZE, 1))

    for _ in tnrange(1, unit='step', desc='inference', leave=False):

        eps = np.random.randn(BATCH_SIZE, NOISE_DIM)
        metrics_inference = inference.train_on_batch(eps, y_tiled)
        
    metrics_dict_inference = dict(zip(inference.metrics_names, 
                                      np.atleast_1d(metrics_inference)))

    global_epoch += 1
    
    # Plot Loss
 
    loss_plot_inference.set_xdata(np.append(loss_plot_inference.get_xdata(),
                                            global_epoch))
    loss_plot_inference.set_ydata(np.append(loss_plot_inference.get_ydata(), 
                                            metrics_dict_inference['loss']))

    loss_plot_inference.set_label('inference ({:.2f})' \
                                  .format(metrics_dict_inference['loss']))

    loss_plot_discrim.set_xdata(np.append(loss_plot_discrim.get_xdata(),
                                          global_epoch))
    loss_plot_discrim.set_ydata(np.append(loss_plot_discrim.get_ydata(),
                                          metrics_dict_discrim['loss']))

    loss_plot_discrim.set_label('discriminator ({:.2f})' \
                                  .format(metrics_dict_discrim['loss']))
    
    ax1.set_xlabel('epoch {:2d}'.format(global_epoch))
    ax1.legend(loc='upper left')
    
    ax1.relim()
    ax1.autoscale_view()
    
    # Contour Plot
    
    ax2.cla()

    w_grid_ratio = ratio_estimator.predict(w_grid.reshape(300*300, 2))
    w_grid_ratio = w_grid_ratio.reshape(300, 300)

    ax2.contourf(w1, w2, w_grid_ratio, cmap='magma')
    ax2.scatter(*inputs.T, c=targets, s=4.**2, alpha=.8, cmap='coolwarm')

    ax2.set_xlabel('$w_1$')
    ax2.set_ylabel('$w_2$')

    ax2.set_xlim(w_min, w_max)
    ax2.set_ylim(w_min, w_max)
    
    # Progress Bar Updates
    
    prog_bar.update()
    prog_bar.set_postfix(loss_inference=metrics_dict_inference['loss'],
                         loss_discriminator=metrics_dict_discrim['loss'])

    return loss_plot_inference, loss_plot_discrim
In [ ]:
with tqdm_notebook(total=50, 
                   unit='epoch', leave=True) as prog_bar:

    anim = FuncAnimation(fig, 
                         train_animate,
                         fargs=(prog_bar,),
                         frames=50,
                         interval=200, # 5 fps
                         blit=True)
    
    anim_html5_video = anim.to_html5_video()
    
HTML(anim_html5_video)
In [ ]:
with tqdm_notebook(total=50, 
                   unit='epoch', leave=True) as prog_bar:

    anim = FuncAnimation(fig, 
                         train_animate,
                         fargs=(prog_bar,),
                         frames=50,
                         interval=200, # 5 fps
                         blit=True)
    
    anim_html5_video = anim.to_html5_video()
    
HTML(anim_html5_video)
In [ ]:
with tqdm_notebook(total=50, 
                   unit='epoch', leave=True) as prog_bar:

    anim = FuncAnimation(fig, 
                         train_animate,
                         fargs=(prog_bar,),
                         frames=50,
                         interval=200, # 5 fps
                         blit=True)
    
    anim_html5_video = anim.to_html5_video()
    
HTML(anim_html5_video)
In [ ]:
with tqdm_notebook(total=50, 
                   unit='epoch', leave=True) as prog_bar:

    anim = FuncAnimation(fig, 
                         train_animate,
                         fargs=(prog_bar,),
                         frames=50,
                         interval=200, # 5 fps
                         blit=True)
    
    anim_html5_video = anim.to_html5_video()
    
HTML(anim_html5_video)

Evaluating the model

In [ ]:
w_sample_prior = prior.rvs(size=128)
eps = np.random.randn(256, NOISE_DIM)
w_sample_posterior = inference.predict(eps)
inputs = np.vstack((w_sample_prior, w_sample_posterior))
targets = np.hstack((np.zeros(128), np.ones(256)))
In [ ]:
w_grid_ratio = ratio_estimator.predict(w_grid.reshape(300*300, 2))
w_grid_ratio = w_grid_ratio.reshape(300, 300)
In [ ]:
fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(9, 4))

ax1.contourf(w1, w2, w_grid_ratio, cmap='magma')
ax1.scatter(*inputs.T, c=targets, s=4.**2, alpha=.8, cmap='coolwarm')

ax1.set_xlabel('$w_1$')

ax1.set_xlim(w_min, w_max)
ax1.set_ylim(w_min, w_max)

ax2.contourf(w1, w2, np.sum(llhs, axis=2), 
             cmap=plt.cm.magma)
ax2.scatter(*inputs.T, c=targets, s=4.**2, alpha=.8, cmap='coolwarm')

ax2.set_xlabel('$w_1$')
ax2.set_ylabel('$w_2$')

ax2.set_xlim(w_min, w_max)
ax2.set_ylim(w_min, w_max)

plt.show()
In [ ]:
eps = np.random.randn(5000, NOISE_DIM)
w_sample_posterior = inference.predict(eps)
In [ ]:
fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(9, 4))

ax1.contourf(w1, w2, 
            np.exp(log_prior+np.sum(llhs, axis=2)), 
            cmap=plt.cm.magma)

ax1.scatter(*inference.predict(eps[::10]).T, 
            s=4.**2, alpha=.6, cmap='coolwarm_r')

ax1.set_xlabel('$w_1$')
ax1.set_ylabel('$w_2$')

ax1.set_xlim(w_min, w_max)
ax1.set_ylim(w_min, w_max)

sns.kdeplot(*inference.predict(eps).T,
            cmap='magma', ax=ax2)

ax2.set_xlim(w_min, w_max)
ax2.set_ylim(w_min, w_max)

plt.show()

Variational Inference with Implicit Approximate Inference Models (WIP Pt. 9)

Comments

In [1]:
%matplotlib inline
%config InlineBackend.figure_format = 'svg'
In [2]:
import numpy as np
import keras.backend as K

import matplotlib.pyplot as plt
import seaborn as sns

from scipy.stats import logistic, multivariate_normal, norm
from scipy.special import expit

from keras.models import Model, Sequential
from keras.layers import Activation, Dense, Dot, Input
from keras.optimizers import Adam
from keras.utils.vis_utils import model_to_dot

from mpl_toolkits.mplot3d import Axes3D
from matplotlib.animation import FuncAnimation

from IPython.display import HTML, SVG, display_html
from tqdm import tnrange, tqdm_notebook

Using TensorFlow backend.
In [3]:
# display animation inline
plt.rc('animation', html='html5')
plt.style.use('seaborn-notebook')
sns.set_context('notebook')
In [4]:
np.set_printoptions(precision=2,
                    edgeitems=3,
                    linewidth=80,
                    suppress=True)
In [5]:
K.tf.__version__
Out[5]:
'1.2.1'
In [6]:
LATENT_DIM = 2
NOISE_DIM = 3
BATCH_SIZE = 200
PRIOR_VARIANCE = 2.
LEARNING_RATE = 3e-3
PRETRAIN_EPOCHS = 60

Bayesian Logistic Regression (Synthetic Data)

In [7]:
w_min, w_max = -5, 5
In [8]:
w1, w2 = np.mgrid[w_min:w_max:300j, w_min:w_max:300j]
In [9]:
w_grid = np.dstack((w1, w2))
w_grid.shape
Out[9]:
(300, 300, 2)
In [10]:
prior = multivariate_normal(mean=np.zeros(LATENT_DIM), 
                            cov=PRIOR_VARIANCE)
In [11]:
log_prior = prior.logpdf(w_grid)
log_prior.shape
Out[11]:
(300, 300)
In [12]:
log_prior = -np.sum(w_grid**2, axis=2)/2/PRIOR_VARIANCE
log_prior.shape
Out[12]:
(300, 300)
In [13]:
fig, ax = plt.subplots(figsize=(5, 5))

ax.contourf(w1, w2, log_prior, cmap='magma')

ax.set_xlabel('$w_1$')
ax.set_ylabel('$w_2$')

ax.set_xlim(w_min, w_max)
ax.set_ylim(w_min, w_max)

plt.show()
In [14]:
x1 = np.array([ 1.5,  1.])
x2 = np.array([-1.5,  1.])
x3 = np.array([  .5, -1.])
In [15]:
X = np.vstack((x1, x2, x3))
X.shape
Out[15]:
(3, 2)
In [16]:
y1 = 1
y2 = 1
y3 = 0
In [17]:
y = np.stack((y1, y2, y3))
y.shape
Out[17]:
(3,)
In [18]:
def log_likelihood(w, x, y):
    # equiv. to negative binary cross entropy
    return np.log(expit(np.dot(w.T, x)*(-1)**(1-y)))
In [19]:
llhs = log_likelihood(w_grid.T, X.T, y)
llhs.shape
Out[19]:
(300, 300, 3)
In [20]:
fig, axes = plt.subplots(ncols=3, nrows=1, figsize=(6, 2))
fig.tight_layout()

for i, ax in enumerate(axes):
    
    ax.contourf(w1, w2, llhs[::,::,i], cmap=plt.cm.magma)

    ax.set_xlim(w_min, w_max)
    ax.set_ylim(w_min, w_max)
    
    ax.set_title('$p(y_{{{0}}} \mid x_{{{0}}}, w)$'.format(i+1))
    ax.set_xlabel('$w_1$')    
    
    if not i:
        ax.set_ylabel('$w_2$')

plt.show()
In [21]:
fig, ax = plt.subplots(figsize=(5, 5))

ax.contourf(w1, w2, np.sum(llhs, axis=2), 
                cmap=plt.cm.magma)

ax.set_xlabel('$w_1$')
ax.set_ylabel('$w_2$')

ax.set_xlim(w_min, w_max)
ax.set_ylim(w_min, w_max)

plt.show()
In [22]:
fig, ax = plt.subplots(figsize=(5, 5))

ax.contourf(w1, w2, 
            np.exp(log_prior+np.sum(llhs, axis=2)), 
            cmap='magma')

ax.scatter(*X.T, c=y, cmap='coolwarm', marker=',')

ax.set_xlabel('$w_1$')
ax.set_ylabel('$w_2$')

ax.set_xlim(w_min, w_max)
ax.set_ylim(w_min, w_max)

plt.show()

Model Definitions

Density Ratio Estimator (Discriminator) Model

$T_{\psi}(x, z)$

Here we consider

$T_{\psi}(w)$

$T_{\psi} : \mathbb{R}^2 \to \mathbb{R}$

In [23]:
discriminator = Sequential(name='discriminator')
discriminator.add(Dense(10, input_dim=LATENT_DIM, activation='relu'))
discriminator.add(Dense(20, activation='relu'))
discriminator.add(Dense(1, activation=None, name='logit'))
discriminator.add(Activation('sigmoid'))
discriminator.compile(optimizer=Adam(lr=LEARNING_RATE),
                      loss='binary_crossentropy',
                      metrics=['binary_accuracy'])
In [24]:
ratio_estimator = Model(
    inputs=discriminator.inputs, 
    outputs=discriminator.get_layer(name='logit').output)
In [25]:
SVG(model_to_dot(discriminator, show_shapes=True)
    .create(prog='dot', format='svg'))
Out[25]:
G 140532566838016 dense_1_input: InputLayerinput:output:(None, 2)(None, 2)140532566553712 dense_1: Denseinput:output:(None, 2)(None, 10)140532566838016->140532566553712 140532566554384 dense_2: Denseinput:output:(None, 10)(None, 20)140532566553712->140532566554384 140532629958784 logit: Denseinput:output:(None, 20)(None, 1)140532566554384->140532629958784 140532567508640 activation_1: Activationinput:output:(None, 1)(None, 1)140532629958784->140532567508640
In [26]:
w_grid_ratio = ratio_estimator.predict(w_grid.reshape(300*300, 2))
w_grid_ratio = w_grid_ratio.reshape(300, 300)

Initial density ratio, prior to any training

In [27]:
fig, ax = plt.subplots(figsize=(5, 5))

ax.contourf(w1, w2, w_grid_ratio, cmap=plt.cm.magma)

ax.set_xlabel('$w_1$')
ax.set_ylabel('$w_2$')

ax.set_xlim(w_min, w_max)
ax.set_ylim(w_min, w_max)

plt.show()
In [28]:
discriminator.evaluate(prior.rvs(size=5), np.zeros(5))

5/5 [==============================] - 0s
Out[28]:
[1.0519158840179443, 0.0]

Approximate Inference Model

$z_{\phi}(x, \epsilon)$

Here we only consider

$z_{\phi}(\epsilon)$

$z_{\phi}: \mathbb{R}^3 \to \mathbb{R}^2$

In [29]:
inference = Sequential()
inference.add(Dense(10, input_dim=NOISE_DIM, activation='relu'))
inference.add(Dense(20, activation='relu'))
inference.add(Dense(LATENT_DIM, activation=None))
inference.summary()

_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
dense_3 (Dense)              (None, 10)                40        
_________________________________________________________________
dense_4 (Dense)              (None, 20)                220       
_________________________________________________________________
dense_5 (Dense)              (None, 2)                 42        
=================================================================
Total params: 302
Trainable params: 302
Non-trainable params: 0
_________________________________________________________________

The variational parameters $\phi$ are the trainable weights of the approximate inference model

In [30]:
phi = inference.trainable_weights
phi
Out[30]:
[<tf.Variable 'dense_3/kernel:0' shape=(3, 10) dtype=float32_ref>,
 <tf.Variable 'dense_3/bias:0' shape=(10,) dtype=float32_ref>,
 <tf.Variable 'dense_4/kernel:0' shape=(10, 20) dtype=float32_ref>,
 <tf.Variable 'dense_4/bias:0' shape=(20,) dtype=float32_ref>,
 <tf.Variable 'dense_5/kernel:0' shape=(20, 2) dtype=float32_ref>,
 <tf.Variable 'dense_5/bias:0' shape=(2,) dtype=float32_ref>]
In [31]:
SVG(model_to_dot(inference, show_shapes=True)
    .create(prog='dot', format='svg'))
Out[31]:
G 140532567173216 dense_3_input: InputLayerinput:output:(None, 3)(None, 3)140532566249824 dense_3: Denseinput:output:(None, 3)(None, 10)140532567173216->140532566249824 140532566253408 dense_4: Denseinput:output:(None, 10)(None, 20)140532566249824->140532566253408 140532567137752 dense_5: Denseinput:output:(None, 20)(None, 2)140532566253408->140532567137752
In [32]:
w_sample_prior = prior.rvs(size=BATCH_SIZE)
w_sample_prior.shape
Out[32]:
(200, 2)
In [33]:
eps = np.random.randn(BATCH_SIZE, NOISE_DIM)
w_sample_posterior = inference.predict(eps)
w_sample_posterior.shape
Out[33]:
(200, 2)
In [34]:
inputs = np.vstack((w_sample_prior, w_sample_posterior))
targets = np.hstack((np.zeros(BATCH_SIZE), np.ones(BATCH_SIZE)))
In [35]:
fig, ax = plt.subplots(figsize=(5, 5))

ax.contourf(w1, w2, 
            np.exp(log_prior+np.sum(llhs, axis=2)), 
            cmap=plt.cm.magma)

ax.scatter(*inputs.T, c=targets, s=4.**2, alpha=.8, cmap='coolwarm')

ax.set_xlabel('$w_1$')
ax.set_ylabel('$w_2$')

ax.set_xlim(w_min, w_max)
ax.set_ylim(w_min, w_max)

plt.show()
In [36]:
metrics = discriminator.evaluate(inputs, targets)

 32/400 [=>............................] - ETA: 0s
In [37]:
w_grid_ratio = ratio_estimator.predict(w_grid.reshape(300*300, 2))
w_grid_ratio = w_grid_ratio.reshape(300, 300)
In [38]:
metrics
Out[38]:
[0.72763251781463623, 0.60250000000000004]
In [39]:
metrics_dict = dict(zip(discriminator.metrics_names, metrics))
In [40]:
fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize=(9, 4))

metrics_plots = {k:ax1.plot([], label=k)[0] 
                 for k in ['loss']} # discriminator.metrics_names}

ax1.set_xlabel('epoch')
ax1.legend(loc='upper left')

ax2.contourf(w1, w2, w_grid_ratio, cmap='magma')
ax2.scatter(*inputs.T, c=targets, s=4.**2, alpha=.8, cmap='coolwarm')

ax2.set_xlabel('$w_1$')
ax2.set_ylabel('$w_2$')

ax2.set_xlim(w_min, w_max)
ax2.set_ylim(w_min, w_max)

plt.show()
Discriminator pre-training
In [41]:
def train_animate(epoch_num, prog_bar, batch_size=200, steps_per_epoch=15):

    # Single training epoch
    
    for step in tnrange(steps_per_epoch, unit='step', leave=False):

        w_sample_prior = prior.rvs(size=batch_size)

        eps = np.random.randn(batch_size, NOISE_DIM)
        w_sample_posterior = inference.predict(eps)

        inputs = np.vstack((w_sample_prior, w_sample_posterior))
        targets = np.hstack((np.zeros(batch_size), np.ones(batch_size)))

        metrics = discriminator.train_on_batch(inputs, targets)

    # Plot Metrics
        
    metrics_dict = dict(zip(discriminator.metrics_names, metrics))

    for metric in metrics_plots:
        metrics_plots[metric].set_xdata(np.append(metrics_plots[metric].get_xdata(), 
                                                  epoch_num))    
        metrics_plots[metric].set_ydata(np.append(metrics_plots[metric].get_ydata(), 
                                                  metrics_dict[metric]))
        metrics_plots[metric].set_label('{} ({:.2f})' \
                                        .format(metric, 
                                                metrics_dict[metric]))
    
    ax1.set_xlabel('epoch {:2d}'.format(epoch_num))
    ax1.legend(loc='upper left')

    ax1.relim()
    ax1.autoscale_view()
    
    # Contour Plot
    
    ax2.cla()

    w_grid_ratio = ratio_estimator.predict(w_grid.reshape(300*300, 2))
    w_grid_ratio = w_grid_ratio.reshape(300, 300)

    ax2.contourf(w1, w2, w_grid_ratio, cmap='magma')
    ax2.scatter(*inputs.T, c=targets, s=4.**2, alpha=.8, cmap='coolwarm')

    ax2.set_xlabel('$w_1$')
    ax2.set_ylabel('$w_2$')

    ax2.set_xlim(w_min, w_max)
    ax2.set_ylim(w_min, w_max)
    
    # Progress Bar Updates
    
    prog_bar.update()
    prog_bar.set_postfix(**metrics_dict)

    return list(metrics_plots.values())
In [42]:
# main training loop is managed by higher-order
# FuncAnimation which makes calls to an `animate` 
# function that encapsulates the logic of single
# training epoch. Has benefit of producing 
# animation but can incur significant overhead
with tqdm_notebook(total=PRETRAIN_EPOCHS, 
                   unit='epoch', leave=True) as prog_bar:

    anim = FuncAnimation(fig, 
                         train_animate,
                         fargs=(prog_bar,),
                         frames=PRETRAIN_EPOCHS,
                         interval=200, # 5 fps
                         blit=True)

    anim_html5_video = anim.to_html5_video()

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.

Widget Javascript not detected.  It may not be installed or enabled properly.



















In [43]:



    


HTML(anim_html5_video)




















Out[43]:

























In [44]:



    


inputs = np.vstack((w_sample_prior, w_sample_posterior))
targets = np.hstack((np.zeros(BATCH_SIZE), np.ones(BATCH_SIZE)))

















In [45]:



    


metrics = discriminator.evaluate(inputs, targets)


























 32/400 [=>............................] - ETA: 0s


















In [46]:



    


w_grid_ratio = ratio_estimator.predict(w_grid.reshape(300*300, 2))
w_grid_ratio = w_grid_ratio.reshape(300, 300)

















In [47]:



    


fig, ax = plt.subplots(figsize=(5, 5))

ax.contourf(w1, w2, w_grid_ratio, cmap='magma')

ax.scatter(*inputs.T, c=targets, s=4.**2, alpha=.8, cmap='coolwarm')

metrics_dict = dict(zip(discriminator.metrics_names, metrics))

props = dict(boxstyle='round', facecolor='w', alpha=0.5)

ax.text(0.05, 0.05, 
        ('accuracy: {binary_accuracy:.2f}\n'        
         'loss: {loss:.2f}').format(**metrics_dict), 
        transform=ax.transAxes, bbox=props)

ax.set_xlabel('$w_1$')
ax.set_ylabel('$w_2$')

ax.set_xlim(w_min, w_max)
ax.set_ylim(w_min, w_max)

plt.show()













































In [48]:



    


fig, ax = plt.subplots(figsize=(5, 5))

ax.contourf(w1, w2, w_grid_ratio, cmap='magma')

ax.scatter(*inputs.T, c=targets, s=4.**2, alpha=.8, cmap='coolwarm')

metrics_dict = dict(zip(discriminator.metrics_names, metrics))

props = dict(boxstyle='round', facecolor='w', alpha=0.5)

ax.text(0.05, 0.05, 
        ('accuracy: {binary_accuracy:.2f}\n'        
         'loss: {loss:.2f}').format(**metrics_dict), 
        transform=ax.transAxes, bbox=props)

ax.set_xlabel('$w_1$')
ax.set_ylabel('$w_2$')

ax.set_xlim(w_min, w_max)
ax.set_ylim(w_min, w_max)

plt.show()



















































Evidence lower bound













In [49]:



    


def set_trainable(model, trainable):
    """inorder traversal"""
    model.trainable = trainable

    if isinstance(model, Model): # i.e. has layers
        for layer in model.layers:
            set_trainable(layer, trainable)

















In [50]:



    


y_pred = K.sigmoid(K.dot(
    K.constant(w_grid),
    K.transpose(K.constant(X))))
y_pred




















Out[50]:








<tf.Tensor 'Sigmoid:0' shape=(300, 300, 3) dtype=float32>



















In [51]:



    


y_true = K.ones((300, 300, 1))*K.constant(y)
y_true




















Out[51]:








<tf.Tensor 'mul_33:0' shape=(300, 300, 3) dtype=float32>



















In [52]:



    


llhs_keras = - K.binary_crossentropy(
                   y_pred, 
                   y_true, 
                   from_logits=False)

















In [53]:



    


sess = K.get_session()

















In [54]:



    


np.allclose(np.sum(llhs, axis=-1),
            sess.run(K.sum(llhs_keras, axis=-1)))




















Out[54]:








True



















In [55]:



    


fig, ax = plt.subplots(figsize=(5, 5))

ax.contourf(w1, w2, sess.run(K.sum(llhs_keras, axis=-1)), 
            cmap=plt.cm.magma)

ax.set_xlabel('$w_1$')
ax.set_ylabel('$w_2$')

ax.set_xlim(w_min, w_max)
ax.set_ylim(w_min, w_max)

plt.show()













































In [56]:



    


def make_elbo(ratio_estimator):
    
    set_trainable(ratio_estimator, False)
    
    def elbo(y_true, w_sample):
        kl_estimate = ratio_estimator(w_sample)
        y_pred = K.dot(w_sample, K.transpose(K.constant(X)))
        log_likelihood = - K.binary_crossentropy(y_pred, y_true, 
                                                 from_logits=True)
        return K.mean(log_likelihood-kl_estimate, axis=-1)
    
    return elbo

















In [57]:



    


inference_loss = lambda y_true, w_sample: -make_elbo(ratio_estimator)(y_true, w_sample)

















In [58]:



    


fig, ax = plt.subplots(figsize=(5, 5))

ax.contourf(w1, w2, sess.run(inference_loss(y_true, K.constant(w_grid))), 
            cmap=plt.cm.magma)

ax.set_xlabel('$w_1$')
ax.set_ylabel('$w_2$')

ax.set_xlim(w_min, w_max)
ax.set_ylim(w_min, w_max)

plt.show()













































In [59]:



    


inference.compile(loss=inference_loss, 
                  optimizer=Adam(lr=LEARNING_RATE))

















In [60]:



    


eps = np.random.randn(BATCH_SIZE, NOISE_DIM)

















In [61]:



    


y_true = K.repeat_elements(K.expand_dims(K.constant(y), axis=0), 
                           axis=0, rep=BATCH_SIZE)
y_true




















Out[61]:








<tf.Tensor 'concat:0' shape=(200, 3) dtype=float32>



















In [62]:



    


sess.run(K.mean(inference_loss(y_true, inference(K.constant(eps))), axis=-1))




















Out[62]:








3.8003495



















In [63]:



    


inference.evaluate(eps, np.tile(y, reps=(BATCH_SIZE, 1)))


























 32/200 [===>..........................] - ETA: 0s










Out[63]:








3.8003493118286134

























Adversarial Training













In [64]:



    


fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize=(9, 4))

global_epoch = 0

loss_plot_inference, = ax1.plot([], label='inference')
loss_plot_discrim, = ax1.plot([], label='discriminator')

ax1.set_xlabel('epoch')
ax1.set_ylabel('loss')
ax1.legend(loc='upper left')

ax2.contourf(w1, w2, w_grid_ratio, cmap='magma')
ax2.scatter(*inputs.T, c=targets, s=4.**2, alpha=.8, cmap='coolwarm')

ax2.set_xlabel('$w_1$')
ax2.set_ylabel('$w_2$')

ax2.set_xlim(w_min, w_max)
ax2.set_ylim(w_min, w_max)

plt.show()













































In [65]:



    


def train_animate(epoch_num, prog_bar, batch_size=200, 
                  steps_per_epoch=15):

    global global_epoch, loss_plot_inference, loss_plot_discrim
    
    # Single training epoch

    ## Ratio estimator training
        
    set_trainable(discriminator, True)

    for _ in tnrange(3*50, unit='step', desc='discriminator', 
                     leave=False):

        w_sample_prior = prior.rvs(size=BATCH_SIZE)

        eps = np.random.randn(BATCH_SIZE, NOISE_DIM)
        w_sample_posterior = inference.predict(eps)

        inputs = np.vstack((w_sample_prior, w_sample_posterior))
        targets = np.hstack((np.zeros(BATCH_SIZE), np.ones(BATCH_SIZE)))

        metrics_discrim = discriminator.train_on_batch(inputs, targets)

    metrics_dict_discrim = dict(zip(discriminator.metrics_names, 
                                    np.atleast_1d(metrics_discrim)))
    
    ## Inference model training

    set_trainable(ratio_estimator, False)

    y_tiled = np.tile(y, reps=(BATCH_SIZE, 1))

    for _ in tnrange(1, unit='step', desc='inference', leave=False):

        eps = np.random.randn(BATCH_SIZE, NOISE_DIM)
        metrics_inference = inference.train_on_batch(eps, y_tiled)

    metrics_dict_inference = dict(zip(inference.metrics_names, 
                                      np.atleast_1d(metrics_inference)))

    global_epoch += 1
    
    # Plot Loss
 
    loss_plot_inference.set_xdata(np.append(loss_plot_inference.get_xdata(),
                                            global_epoch))
    loss_plot_inference.set_ydata(np.append(loss_plot_inference.get_ydata(), 
                                            metrics_dict_inference['loss']))

    loss_plot_inference.set_label('inference ({:.2f})' \
                                  .format(metrics_dict_inference['loss']))

    loss_plot_discrim.set_xdata(np.append(loss_plot_discrim.get_xdata(),
                                          global_epoch))
    loss_plot_discrim.set_ydata(np.append(loss_plot_discrim.get_ydata(),
                                          metrics_dict_discrim['loss']))

    loss_plot_discrim.set_label('discriminator ({:.2f})' \
                                  .format(metrics_dict_discrim['loss']))
    
    ax1.set_xlabel('epoch {:2d}'.format(global_epoch))
    ax1.legend(loc='upper left')
    
    ax1.relim()
    ax1.autoscale_view()
    
    # Contour Plot
    
    ax2.cla()

    w_grid_ratio = ratio_estimator.predict(w_grid.reshape(300*300, 2))
    w_grid_ratio = w_grid_ratio.reshape(300, 300)

    ax2.contourf(w1, w2, w_grid_ratio, cmap='magma')
    ax2.scatter(*inputs.T, c=targets, s=4.**2, alpha=.8, cmap='coolwarm')

    ax2.set_xlabel('$w_1$')
    ax2.set_ylabel('$w_2$')

    ax2.set_xlim(w_min, w_max)
    ax2.set_ylim(w_min, w_max)
    
    # Progress Bar Updates
    
    prog_bar.update()
    prog_bar.set_postfix(loss_inference=metrics_dict_inference['loss'],
                         loss_discriminator=metrics_dict_discrim['loss'])

    return loss_plot_inference, loss_plot_discrim

















In [66]:



    


with tqdm_notebook(total=50, 
                   unit='epoch', leave=True) as prog_bar:

    anim = FuncAnimation(fig, 
                         train_animate,
                         fargs=(prog_bar,),
                         frames=50,
                         interval=200, # 5 fps
                         blit=True)
    
    anim_html5_video = anim.to_html5_video()
    
HTML(anim_html5_video)


























Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 































Out[66]:

























In [67]:



    


with tqdm_notebook(total=50, 
                   unit='epoch', leave=True) as prog_bar:

    anim = FuncAnimation(fig, 
                         train_animate,
                         fargs=(prog_bar,),
                         frames=50,
                         interval=200, # 5 fps
                         blit=True)
    
    anim_html5_video = anim.to_html5_video()
    
HTML(anim_html5_video)


























Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 































Out[67]:

























In [68]:



    


with tqdm_notebook(total=50, 
                   unit='epoch', leave=True) as prog_bar:

    anim = FuncAnimation(fig, 
                         train_animate,
                         fargs=(prog_bar,),
                         frames=50,
                         interval=200, # 5 fps
                         blit=True)
    
    anim_html5_video = anim.to_html5_video()
    
HTML(anim_html5_video)


























Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 































Out[68]:

























In [69]:



    


with tqdm_notebook(total=50, 
                   unit='epoch', leave=True) as prog_bar:

    anim = FuncAnimation(fig, 
                         train_animate,
                         fargs=(prog_bar,),
                         frames=50,
                         interval=200, # 5 fps
                         blit=True)
    
    anim_html5_video = anim.to_html5_video()
    
HTML(anim_html5_video)


























Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 































Out[69]:































Evaluating the model













In [70]:



    


w_sample_prior = prior.rvs(size=128)
eps = np.random.randn(256, NOISE_DIM)
w_sample_posterior = inference.predict(eps)
inputs = np.vstack((w_sample_prior, w_sample_posterior))
targets = np.hstack((np.zeros(128), np.ones(256)))

















In [71]:



    


w_grid_ratio = ratio_estimator.predict(w_grid.reshape(300*300, 2))
w_grid_ratio = w_grid_ratio.reshape(300, 300)

















In [72]:



    


fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(9, 4))

ax1.contourf(w1, w2, w_grid_ratio, cmap='magma')
ax1.scatter(*inputs.T, c=targets, s=4.**2, alpha=.8, cmap='coolwarm')

ax1.set_xlabel('$w_1$')

ax1.set_xlim(w_min, w_max)
ax1.set_ylim(w_min, w_max)

ax2.contourf(w1, w2, np.sum(llhs, axis=2), 
             cmap=plt.cm.magma)
ax2.scatter(*inputs.T, c=targets, s=4.**2, alpha=.8, cmap='coolwarm')

ax2.set_xlabel('$w_1$')
ax2.set_ylabel('$w_2$')

ax2.set_xlim(w_min, w_max)
ax2.set_ylim(w_min, w_max)

plt.show()













































In [73]:



    


eps = np.random.randn(5000, NOISE_DIM)
w_sample_posterior = inference.predict(eps)

















In [74]:



    


fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(9, 4))

ax1.contourf(w1, w2, 
            np.exp(log_prior+np.sum(llhs, axis=2)), 
            cmap=plt.cm.magma)

ax1.scatter(*inference.predict(eps[::10]).T, 
            s=4.**2, alpha=.6, cmap='coolwarm_r')

ax1.set_xlabel('$w_1$')
ax1.set_ylabel('$w_2$')

ax1.set_xlim(w_min, w_max)
ax1.set_ylim(w_min, w_max)

sns.kdeplot(*inference.predict(eps).T,
            cmap='magma', ax=ax2)

ax2.set_xlim(w_min, w_max)
ax2.set_ylim(w_min, w_max)

plt.show()













































In [75]:



    


output = expit(np.random.randn(256))
target = np.hstack((np.zeros(128), np.ones(128)))

















In [76]:



    


2*K.mean(K.binary_crossentropy(output=K.constant(output), 
                      target=K.constant(target))).eval(session=sess)




















Out[76]:








1.5870189666748047



















In [77]:



    


np.mean(-np.log(output[128:])-np.log(1-output[:128]))




















Out[77]:








1.5870191065045991



















In [78]:



    


(-np.log(output[:128])-np.log(1-output[128:])).shape




















Out[78]:








(128,)



















In [79]:



    


p1[:128][0]


























---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-79-3424496d14cb> in <module>()
----> 1 p1[:128][0]

NameError: name 'p1' is not defined


















In [ ]:



    


p1[128:][0]

















In [ ]:



    


ratio_estimator.get_weights()[0]












    

  


    

    
Variational Inference with Implicit Approximate Inference Models (WIP Pt. 8)

        

            

            

                
            Comments


        



    

    

    







In [1]:



    


%matplotlib inline
%config InlineBackend.figure_format = 'svg'

















In [2]:



    


import numpy as np
import keras.backend as K

import matplotlib.pyplot as plt
import seaborn as sns

from scipy.stats import logistic, multivariate_normal, norm
from scipy.special import expit

from keras.models import Model, Sequential
from keras.layers import Activation, Dense, Dot, Input
from keras.optimizers import Adam
from keras.utils.vis_utils import model_to_dot

from mpl_toolkits.mplot3d import Axes3D
from matplotlib.animation import FuncAnimation

from IPython.display import HTML, SVG, display_html
from tqdm import tnrange, tqdm_notebook


























Using TensorFlow backend.



















In [3]:



    


# display animation inline
plt.rc('animation', html='html5')
plt.style.use('seaborn-notebook')
sns.set_context('notebook')

















In [4]:



    


np.set_printoptions(precision=2,
                    edgeitems=3,
                    linewidth=80,
                    suppress=True)

















In [5]:



    


K.tf.__version__




















Out[5]:








'1.2.1'



















In [6]:



    


LATENT_DIM = 2
NOISE_DIM = 3
BATCH_SIZE = 200
PRIOR_VARIANCE = 2.
LEARNING_RATE = 3e-3
PRETRAIN_EPOCHS = 60























Bayesian Logistic Regression (Synthetic Data)













In [7]:



    


w_min, w_max = -5, 5

















In [8]:



    


w1, w2 = np.mgrid[w_min:w_max:300j, w_min:w_max:300j]

















In [9]:



    


w_grid = np.dstack((w1, w2))
w_grid.shape




















Out[9]:








(300, 300, 2)



















In [10]:



    


prior = multivariate_normal(mean=np.zeros(LATENT_DIM), 
                            cov=PRIOR_VARIANCE)

















In [11]:



    


log_prior = prior.logpdf(w_grid)
log_prior.shape




















Out[11]:








(300, 300)



















In [12]:



    


fig, ax = plt.subplots(figsize=(5, 5))

ax.contourf(w1, w2, log_prior, cmap='magma')

ax.set_xlabel('$w_1$')
ax.set_ylabel('$w_2$')

ax.set_xlim(w_min, w_max)
ax.set_ylim(w_min, w_max)

plt.show()













































In [13]:



    


x1 = np.array([ 1.5,  1.])
x2 = np.array([-1.5,  1.])
x3 = np.array([  .5, -1.])

















In [14]:



    


X = np.vstack((x1, x2, x3))
X.shape




















Out[14]:








(3, 2)



















In [15]:



    


y1 = 1
y2 = 1
y3 = 0

















In [16]:



    


y = np.stack((y1, y2, y3))
y.shape




















Out[16]:








(3,)



















In [17]:



    


def log_likelihood(w, x, y):
    # equiv. to negative binary cross entropy
    return np.log(expit(np.dot(w.T, x)*(-1)**(1-y)))

















In [18]:



    


llhs = log_likelihood(w_grid.T, X.T, y)
llhs.shape




















Out[18]:








(300, 300, 3)



















In [19]:



    


fig, axes = plt.subplots(ncols=3, nrows=1, figsize=(6, 2))
fig.tight_layout()

for i, ax in enumerate(axes):
    
    ax.contourf(w1, w2, llhs[::,::,i], cmap=plt.cm.magma)

    ax.set_xlim(w_min, w_max)
    ax.set_ylim(w_min, w_max)
    
    ax.set_title('$p(y_{{{0}}} \mid x_{{{0}}}, w)$'.format(i+1))
    ax.set_xlabel('$w_1$')    
    
    if not i:
        ax.set_ylabel('$w_2$')

plt.show()













































In [20]:



    


fig, ax = plt.subplots(figsize=(5, 5))

ax.contourf(w1, w2, np.sum(llhs, axis=2), 
                cmap=plt.cm.magma)

ax.set_xlabel('$w_1$')
ax.set_ylabel('$w_2$')

ax.set_xlim(w_min, w_max)
ax.set_ylim(w_min, w_max)

plt.show()













































In [21]:



    


fig, ax = plt.subplots(figsize=(5, 5))

ax.contourf(w1, w2, 
            np.exp(log_prior+np.sum(llhs, axis=2)), 
            cmap='magma')

ax.scatter(*X.T, c=y, cmap='coolwarm', marker=',')

ax.set_xlabel('$w_1$')
ax.set_ylabel('$w_2$')

ax.set_xlim(w_min, w_max)
ax.set_ylim(w_min, w_max)

plt.show()



















































Model Definitions



















Density Ratio Estimator (Discriminator) Model



















$T_{\psi}(x, z)$


Here we consider


$T_{\psi}(w)$


$T_{\psi} : \mathbb{R}^2 \to \mathbb{R}$













In [22]:



    


discriminator = Sequential(name='discriminator')
discriminator.add(Dense(10, input_dim=LATENT_DIM, activation='relu'))
discriminator.add(Dense(20, activation='relu'))
discriminator.add(Dense(1, activation=None, name='logit'))
discriminator.add(Activation('sigmoid'))
discriminator.compile(optimizer=Adam(lr=LEARNING_RATE),
                      loss='binary_crossentropy',
                      metrics=['binary_accuracy'])

















In [23]:



    


ratio_estimator = Model(
    inputs=discriminator.inputs, 
    outputs=discriminator.get_layer(name='logit').output)

















In [24]:



    


SVG(model_to_dot(discriminator, show_shapes=True)
    .create(prog='dot', format='svg'))




















Out[24]:






G
139996050319736
dense_1_input: InputLayerinput:output:(None, 2)(None, 2)139996049881464
dense_1: Denseinput:output:(None, 2)(None, 10)139996050319736->139996049881464
139996049882416
dense_2: Denseinput:output:(None, 10)(None, 20)139996049881464->139996049882416
139996050321304
logit: Denseinput:output:(None, 20)(None, 1)139996049882416->139996050321304
139996050741624
activation_1: Activationinput:output:(None, 1)(None, 1)139996050321304->139996050741624



















In [25]:



    


w_grid_ratio = ratio_estimator.predict(w_grid.reshape(300*300, 2))
w_grid_ratio = w_grid_ratio.reshape(300, 300)























Initial density ratio, prior to any training













In [26]:



    


fig, ax = plt.subplots(figsize=(5, 5))

ax.contourf(w1, w2, w_grid_ratio, cmap=plt.cm.magma)

ax.set_xlabel('$w_1$')
ax.set_ylabel('$w_2$')

ax.set_xlim(w_min, w_max)
ax.set_ylim(w_min, w_max)

plt.show()













































In [27]:



    


discriminator.evaluate(prior.rvs(size=5), np.zeros(5))


























5/5 [==============================] - 0s











Out[27]:








[0.62235820293426514, 0.60000002384185791]

























Approximate Inference Model



















$z_{\phi}(x, \epsilon)$


Here we only consider


$z_{\phi}(\epsilon)$


$z_{\phi}: \mathbb{R}^3 \to \mathbb{R}^2$













In [28]:



    


inference = Sequential()
inference.add(Dense(10, input_dim=NOISE_DIM, activation='relu'))
inference.add(Dense(20, activation='relu'))
inference.add(Dense(LATENT_DIM, activation=None))
inference.summary()


























_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
dense_3 (Dense)              (None, 10)                40        
_________________________________________________________________
dense_4 (Dense)              (None, 20)                220       
_________________________________________________________________
dense_5 (Dense)              (None, 2)                 42        
=================================================================
Total params: 302
Trainable params: 302
Non-trainable params: 0
_________________________________________________________________

























The variational parameters $\phi$ are the trainable weights of the approximate inference model













In [29]:



    


phi = inference.trainable_weights
phi




















Out[29]:








[<tf.Variable 'dense_3/kernel:0' shape=(3, 10) dtype=float32_ref>,
 <tf.Variable 'dense_3/bias:0' shape=(10,) dtype=float32_ref>,
 <tf.Variable 'dense_4/kernel:0' shape=(10, 20) dtype=float32_ref>,
 <tf.Variable 'dense_4/bias:0' shape=(20,) dtype=float32_ref>,
 <tf.Variable 'dense_5/kernel:0' shape=(20, 2) dtype=float32_ref>,
 <tf.Variable 'dense_5/bias:0' shape=(2,) dtype=float32_ref>]



















In [30]:



    


SVG(model_to_dot(inference, show_shapes=True)
    .create(prog='dot', format='svg'))




















Out[30]:






G
139996049667352
dense_3_input: InputLayerinput:output:(None, 3)(None, 3)139996049443528
dense_3: Denseinput:output:(None, 3)(None, 10)139996049667352->139996049443528
139996049443416
dense_4: Denseinput:output:(None, 10)(None, 20)139996049443528->139996049443416
139996049667520
dense_5: Denseinput:output:(None, 20)(None, 2)139996049443416->139996049667520



















In [31]:



    


w_sample_prior = prior.rvs(size=BATCH_SIZE)
w_sample_prior.shape




















Out[31]:








(200, 2)



















In [32]:



    


eps = np.random.randn(BATCH_SIZE, NOISE_DIM)
w_sample_posterior = inference.predict(eps)
w_sample_posterior.shape




















Out[32]:








(200, 2)



















In [33]:



    


inputs = np.vstack((w_sample_prior, w_sample_posterior))
targets = np.hstack((np.zeros(BATCH_SIZE), np.ones(BATCH_SIZE)))

















In [34]:



    


fig, ax = plt.subplots(figsize=(5, 5))

ax.contourf(w1, w2, 
            np.exp(log_prior+np.sum(llhs, axis=2)), 
            cmap=plt.cm.magma)

ax.scatter(*inputs.T, c=targets, s=4.**2, alpha=.8, cmap='coolwarm')

ax.set_xlabel('$w_1$')
ax.set_ylabel('$w_2$')

ax.set_xlim(w_min, w_max)
ax.set_ylim(w_min, w_max)

plt.show()













































In [35]:



    


metrics = discriminator.evaluate(inputs, targets)


























 32/400 [=>............................] - ETA: 0s


















In [36]:



    


w_grid_ratio = ratio_estimator.predict(w_grid.reshape(300*300, 2))
w_grid_ratio = w_grid_ratio.reshape(300, 300)

















In [37]:



    


metrics




















Out[37]:








[0.69308857917785649, 0.47749999999999998]



















In [38]:



    


metrics_dict = dict(zip(discriminator.metrics_names, metrics))

















In [39]:



    


fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize=(9, 4))

metrics_plots = {k:ax1.plot([], label=k)[0] 
                 for k in ['loss']} # discriminator.metrics_names}

ax1.set_xlabel('epoch')
ax1.legend(loc='upper left')

ax2.contourf(w1, w2, w_grid_ratio, cmap='magma')
ax2.scatter(*inputs.T, c=targets, s=4.**2, alpha=.8, cmap='coolwarm')

ax2.set_xlabel('$w_1$')
ax2.set_ylabel('$w_2$')

ax2.set_xlim(w_min, w_max)
ax2.set_ylim(w_min, w_max)

plt.show()



















































Discriminator pre-training













In [40]:



    


def train_animate(epoch_num, prog_bar, batch_size=200, steps_per_epoch=15):

    # Single training epoch
    
    for step in tnrange(steps_per_epoch, unit='step', leave=False):

        w_sample_prior = prior.rvs(size=batch_size)

        eps = np.random.randn(batch_size, NOISE_DIM)
        w_sample_posterior = inference.predict(eps)

        inputs = np.vstack((w_sample_prior, w_sample_posterior))
        targets = np.hstack((np.zeros(batch_size), np.ones(batch_size)))

        metrics = discriminator.train_on_batch(inputs, targets)

    # Plot Metrics
        
    metrics_dict = dict(zip(discriminator.metrics_names, metrics))

    for metric in metrics_plots:
        metrics_plots[metric].set_xdata(np.append(metrics_plots[metric].get_xdata(), 
                                                  epoch_num))    
        metrics_plots[metric].set_ydata(np.append(metrics_plots[metric].get_ydata(), 
                                                  metrics_dict[metric]))
        metrics_plots[metric].set_label('{} ({:.2f})' \
                                        .format(metric, 
                                                metrics_dict[metric]))
    
    ax1.set_xlabel('epoch {:2d}'.format(epoch_num))
    ax1.legend(loc='upper left')

    ax1.relim()
    ax1.autoscale_view()
    
    # Contour Plot
    
    ax2.cla()

    w_grid_ratio = ratio_estimator.predict(w_grid.reshape(300*300, 2))
    w_grid_ratio = w_grid_ratio.reshape(300, 300)

    ax2.contourf(w1, w2, w_grid_ratio, cmap='magma')
    ax2.scatter(*inputs.T, c=targets, s=4.**2, alpha=.8, cmap='coolwarm')

    ax2.set_xlabel('$w_1$')
    ax2.set_ylabel('$w_2$')

    ax2.set_xlim(w_min, w_max)
    ax2.set_ylim(w_min, w_max)
    
    # Progress Bar Updates
    
    prog_bar.update()
    prog_bar.set_postfix(**metrics_dict)

    return list(metrics_plots.values())

















In [41]:



    


# main training loop is managed by higher-order
# FuncAnimation which makes calls to an `animate` 
# function that encapsulates the logic of single
# training epoch. Has benefit of producing 
# animation but can incur significant overhead
with tqdm_notebook(total=PRETRAIN_EPOCHS, 
                   unit='epoch', leave=True) as prog_bar:

    anim = FuncAnimation(fig, 
                         train_animate,
                         fargs=(prog_bar,),
                         frames=PRETRAIN_EPOCHS,
                         interval=200, # 5 fps
                         blit=True)

    anim_html5_video = anim.to_html5_video()


























Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 







































In [42]:



    


HTML(anim_html5_video)




















Out[42]:

























In [43]:



    


inputs = np.vstack((w_sample_prior, w_sample_posterior))
targets = np.hstack((np.zeros(BATCH_SIZE), np.ones(BATCH_SIZE)))

















In [44]:



    


metrics = discriminator.evaluate(inputs, targets)


























 32/400 [=>............................] - ETA: 0s


















In [45]:



    


w_grid_ratio = ratio_estimator.predict(w_grid.reshape(300*300, 2))
w_grid_ratio = w_grid_ratio.reshape(300, 300)

















In [46]:



    


fig, ax = plt.subplots(figsize=(5, 5))

ax.contourf(w1, w2, w_grid_ratio, cmap='magma')

ax.scatter(*inputs.T, c=targets, s=4.**2, alpha=.8, cmap='coolwarm')

metrics_dict = dict(zip(discriminator.metrics_names, metrics))

props = dict(boxstyle='round', facecolor='w', alpha=0.5)

ax.text(0.05, 0.05, 
        ('accuracy: {binary_accuracy:.2f}\n'        
         'loss: {loss:.2f}').format(**metrics_dict), 
        transform=ax.transAxes, bbox=props)

ax.set_xlabel('$w_1$')
ax.set_ylabel('$w_2$')

ax.set_xlim(w_min, w_max)
ax.set_ylim(w_min, w_max)

plt.show()



















































Evidence lower bound













In [47]:



    


def set_trainable(model, trainable):
    """inorder traversal"""
    model.trainable = trainable

    if isinstance(model, Model): # i.e. has layers
        for layer in model.layers:
            set_trainable(layer, trainable)

















In [48]:



    


y_pred = K.sigmoid(K.dot(
    K.constant(w_grid),
    K.transpose(K.constant(X))))
y_pred




















Out[48]:








<tf.Tensor 'Sigmoid:0' shape=(300, 300, 3) dtype=float32>



















In [49]:



    


y_true = K.ones((300, 300, 1))*K.constant(y)
y_true




















Out[49]:








<tf.Tensor 'mul_33:0' shape=(300, 300, 3) dtype=float32>



















In [50]:



    


llhs_keras = - K.binary_crossentropy(
                   y_pred, 
                   y_true, 
                   from_logits=False)

















In [51]:



    


sess = K.get_session()

















In [52]:



    


np.allclose(np.sum(llhs, axis=-1),
            sess.run(K.sum(llhs_keras, axis=-1)))




















Out[52]:








True



















In [53]:



    


fig, ax = plt.subplots(figsize=(5, 5))

ax.contourf(w1, w2, sess.run(K.sum(llhs_keras, axis=-1)), 
            cmap=plt.cm.magma)

ax.set_xlabel('$w_1$')
ax.set_ylabel('$w_2$')

ax.set_xlim(w_min, w_max)
ax.set_ylim(w_min, w_max)

plt.show()













































In [54]:



    


def make_elbo(ratio_estimator):
    
    set_trainable(ratio_estimator, False)
    
    def elbo(y_true, w_sample):
        kl_estimate = ratio_estimator(w_sample)
        y_pred = K.dot(w_sample, K.transpose(K.constant(X)))
        log_likelihood = - K.binary_crossentropy(y_pred, y_true, 
                                                 from_logits=True)
        return K.mean(log_likelihood-kl_estimate, axis=-1)

    return elbo

















In [55]:



    


elbo = make_elbo(ratio_estimator)

















In [56]:



    


fig, ax = plt.subplots(figsize=(5, 5))

ax.contourf(w1, w2, sess.run(elbo(y_true, K.constant(w_grid))), 
            cmap=plt.cm.magma)

ax.set_xlabel('$w_1$')
ax.set_ylabel('$w_2$')

ax.set_xlim(w_min, w_max)
ax.set_ylim(w_min, w_max)

plt.show()













































In [57]:



    


inference_loss = lambda y_true, w_sample: -make_elbo(ratio_estimator)(y_true, w_sample)

















In [58]:



    


inference.compile(loss=inference_loss, 
                  optimizer=Adam(lr=LEARNING_RATE))

















In [59]:



    


eps = np.random.randn(BATCH_SIZE, NOISE_DIM)

















In [60]:



    


y_true = K.repeat_elements(K.expand_dims(K.constant(y), axis=0), 
                           axis=0, rep=BATCH_SIZE)
y_true




















Out[60]:








<tf.Tensor 'concat:0' shape=(200, 3) dtype=float32>



















In [61]:



    


sess.run(K.mean(elbo(y_true, inference(K.constant(eps))), axis=-1))




















Out[61]:








-2.5127466



















In [62]:



    


inference.evaluate(eps, np.tile(y, reps=(BATCH_SIZE, 1)))


























 32/200 [===>..........................] - ETA: 0s










Out[62]:








2.5127464294433595

























Adversarial Training













In [63]:



    


fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize=(9, 4))

global_epoch = 0

loss_plot_inference, = ax1.plot([], label='inference')
loss_plot_discrim, = ax1.plot([], label='discriminator')

ax1.set_xlabel('epoch')
ax1.set_ylabel('loss')
ax1.legend(loc='upper left')

ax2.contourf(w1, w2, w_grid_ratio, cmap='magma')
ax2.scatter(*inputs.T, c=targets, s=4.**2, alpha=.8, cmap='coolwarm')

ax2.set_xlabel('$w_1$')
ax2.set_ylabel('$w_2$')

ax2.set_xlim(w_min, w_max)
ax2.set_ylim(w_min, w_max)

plt.show()













































In [64]:



    


def train_animate(epoch_num, prog_bar, batch_size=200, 
                  steps_per_epoch=15):

    global global_epoch, loss_plot_inference, loss_plot_discrim
    
    # Single training epoch

    ## Ratio estimator training
        
    set_trainable(discriminator, True)

    for _ in tnrange(3*50, unit='step', desc='discriminator', 
                     leave=False):

        w_sample_prior = prior.rvs(size=BATCH_SIZE)

        eps = np.random.randn(BATCH_SIZE, NOISE_DIM)
        w_sample_posterior = inference.predict(eps)

        inputs = np.vstack((w_sample_prior, w_sample_posterior))
        targets = np.hstack((np.zeros(BATCH_SIZE), np.ones(BATCH_SIZE)))

        metrics_discrim = discriminator.train_on_batch(inputs, targets)

    metrics_dict_discrim = dict(zip(discriminator.metrics_names, 
                                    np.atleast_1d(metrics_discrim)))
    
    ## Inference model training
    
    set_trainable(ratio_estimator, False)

    y_tiled = np.tile(y, reps=(BATCH_SIZE, 1))

    for _ in tnrange(1, unit='step', desc='inference', leave=False):

        eps = np.random.randn(BATCH_SIZE, NOISE_DIM)
        metrics_inference = inference.train_on_batch(eps, y_tiled)
        
    metrics_dict_inference = dict(zip(inference.metrics_names, 
                                      np.atleast_1d(metrics_inference)))

    global_epoch += 1
    
    # Plot Loss
 
    loss_plot_inference.set_xdata(np.append(loss_plot_inference.get_xdata(),
                                            global_epoch))
    loss_plot_inference.set_ydata(np.append(loss_plot_inference.get_ydata(), 
                                            metrics_dict_inference['loss']))

    loss_plot_inference.set_label('inference ({:.2f})' \
                                  .format(metrics_dict_inference['loss']))

    loss_plot_discrim.set_xdata(np.append(loss_plot_discrim.get_xdata(),
                                          global_epoch))
    loss_plot_discrim.set_ydata(np.append(loss_plot_discrim.get_ydata(),
                                          metrics_dict_discrim['loss']))

    loss_plot_discrim.set_label('discriminator ({:.2f})' \
                                  .format(metrics_dict_discrim['loss']))
    
    ax1.set_xlabel('epoch {:2d}'.format(global_epoch))
    ax1.legend(loc='upper left')
    
    ax1.relim()
    ax1.autoscale_view()
    
    # Contour Plot
    
    ax2.cla()

    w_grid_ratio = ratio_estimator.predict(w_grid.reshape(300*300, 2))
    w_grid_ratio = w_grid_ratio.reshape(300, 300)

    ax2.contourf(w1, w2, w_grid_ratio, cmap='magma')
    ax2.scatter(*inputs.T, c=targets, s=4.**2, alpha=.8, cmap='coolwarm')

    ax2.set_xlabel('$w_1$')
    ax2.set_ylabel('$w_2$')

    ax2.set_xlim(w_min, w_max)
    ax2.set_ylim(w_min, w_max)
    
    # Progress Bar Updates
    
    prog_bar.update()
    prog_bar.set_postfix(loss_inference=metrics_dict_inference['loss'],
                         loss_discriminator=metrics_dict_discrim['loss'])

    return loss_plot_inference, loss_plot_discrim

















In [65]:



    


with tqdm_notebook(total=50, 
                   unit='epoch', leave=True) as prog_bar:

    anim = FuncAnimation(fig, 
                         train_animate,
                         fargs=(prog_bar,),
                         frames=50,
                         interval=200, # 5 fps
                         blit=True)
    
    anim_html5_video = anim.to_html5_video()
    
HTML(anim_html5_video)


























Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 































Out[65]:

























In [66]:



    


with tqdm_notebook(total=50, 
                   unit='epoch', leave=True) as prog_bar:

    anim = FuncAnimation(fig, 
                         train_animate,
                         fargs=(prog_bar,),
                         frames=50,
                         interval=200, # 5 fps
                         blit=True)
    
    anim_html5_video = anim.to_html5_video()
    
HTML(anim_html5_video)


























Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 































Out[66]:

























In [67]:



    


with tqdm_notebook(total=50, 
                   unit='epoch', leave=True) as prog_bar:

    anim = FuncAnimation(fig, 
                         train_animate,
                         fargs=(prog_bar,),
                         frames=50,
                         interval=200, # 5 fps
                         blit=True)
    
    anim_html5_video = anim.to_html5_video()
    
HTML(anim_html5_video)


























Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 































Out[67]:

























In [68]:



    


with tqdm_notebook(total=50, 
                   unit='epoch', leave=True) as prog_bar:

    anim = FuncAnimation(fig, 
                         train_animate,
                         fargs=(prog_bar,),
                         frames=50,
                         interval=200, # 5 fps
                         blit=True)
    
    anim_html5_video = anim.to_html5_video()
    
HTML(anim_html5_video)


























Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 





















Widget Javascript not detected.  It may not be installed or enabled properly.

















 
 































Out[68]:































Evaluating the model













In [81]:



    


w_sample_prior = prior.rvs(size=128)
eps = np.random.randn(256, NOISE_DIM)
w_sample_posterior = inference.predict(eps)
inputs = np.vstack((w_sample_prior, w_sample_posterior))
targets = np.hstack((np.zeros(128), np.ones(256)))

















In [82]:



    


w_grid_ratio = ratio_estimator.predict(w_grid.reshape(300*300, 2))
w_grid_ratio = w_grid_ratio.reshape(300, 300)

















In [85]:



    


fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(9, 4))

ax1.contourf(w1, w2, w_grid_ratio, cmap='magma')
ax1.scatter(*inputs.T, c=targets, s=4.**2, alpha=.8, cmap='coolwarm')

ax1.set_xlabel('$w_1$')

ax1.set_xlim(w_min, w_max)
ax1.set_ylim(w_min, w_max)

ax2.contourf(w1, w2, np.sum(llhs, axis=2), 
             cmap=plt.cm.magma)
ax2.scatter(*inputs.T, c=targets, s=4.**2, alpha=.8, cmap='coolwarm')

ax2.set_xlabel('$w_1$')
ax2.set_ylabel('$w_2$')

ax2.set_xlim(w_min, w_max)
ax2.set_ylim(w_min, w_max)

plt.show()













































In [101]:



    


eps = np.random.randn(5000, NOISE_DIM)
w_sample_posterior = inference.predict(eps)

















In [105]:



    


fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(9, 4))

ax1.contourf(w1, w2, 
            np.exp(log_prior+np.sum(llhs, axis=2)), 
            cmap=plt.cm.magma)

ax1.scatter(*inference.predict(eps[::10]).T, 
            s=4.**2, alpha=.6, cmap='coolwarm_r')

ax1.set_xlabel('$w_1$')
ax1.set_ylabel('$w_2$')

ax1.set_xlim(w_min, w_max)
ax1.set_ylim(w_min, w_max)

sns.kdeplot(*inference.predict(eps).T,
            cmap='magma', ax=ax2)

plt.show()